- * {@link org.apache.dolphinscheduler.common.datasource.clickhouse.ClickhouseConnectionParam} - * {@link org.apache.dolphinscheduler.common.datasource.db2.Db2ConnectionParam} - * {@link org.apache.dolphinscheduler.common.datasource.hive.HiveConnectionParam} - * {@link org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam} - * {@link org.apache.dolphinscheduler.common.datasource.oracle.OracleConnectionParam} - * {@link org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlConnectionParam} - * {@link org.apache.dolphinscheduler.common.datasource.presto.PrestoConnectionParam} - * {@link org.apache.dolphinscheduler.common.datasource.spark.SparkConnectionParam} - * {@link org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerConnectionParam} - */ -@JsonInclude(Include.NON_NULL) -public abstract class BaseConnectionParam implements ConnectionParam { - - protected String user; - - protected String password; - - protected String address; - - protected String database; - - protected String jdbcUrl; - - protected String other; - - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public String getAddress() { - return address; - } - - public void setAddress(String address) { - this.address = address; - } - - public String getDatabase() { - return database; - } - - public void setDatabase(String database) { - this.database = database; - } - - public String getJdbcUrl() { - return jdbcUrl; - } - - public void setJdbcUrl(String jdbcUrl) { - this.jdbcUrl = jdbcUrl; - } - - public String getOther() { - return other; - } - - public void setOther(String other) { - this.other = other; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsConnectionParam.java deleted file mode 100644 index 908b7c8cef..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsConnectionParam.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource; - -public class BaseHdfsConnectionParam extends BaseConnectionParam { - protected String principal; - protected String javaSecurityKrb5Conf; - protected String loginUserKeytabUsername; - protected String loginUserKeytabPath; - - public String getPrincipal() { - return principal; - } - - public void setPrincipal(String principal) { - this.principal = principal; - } - - public String getJavaSecurityKrb5Conf() { - return javaSecurityKrb5Conf; - } - - public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) { - this.javaSecurityKrb5Conf = javaSecurityKrb5Conf; - } - - public String getLoginUserKeytabUsername() { - return loginUserKeytabUsername; - } - - public void setLoginUserKeytabUsername(String loginUserKeytabUsername) { - this.loginUserKeytabUsername = loginUserKeytabUsername; - } - - public String getLoginUserKeytabPath() { - return loginUserKeytabPath; - } - - public void setLoginUserKeytabPath(String loginUserKeytabPath) { - this.loginUserKeytabPath = loginUserKeytabPath; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsDatasourceParamDTO.java deleted file mode 100644 index 747d3ea274..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsDatasourceParamDTO.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource; - -public abstract class BaseHdfsDatasourceParamDTO extends BaseDataSourceParamDTO { - - protected String principal; - - protected String javaSecurityKrb5Conf; - - protected String loginUserKeytabUsername; - - protected String loginUserKeytabPath; - - public String getPrincipal() { - return principal; - } - - public void setPrincipal(String principal) { - this.principal = principal; - } - - public String getLoginUserKeytabUsername() { - return loginUserKeytabUsername; - } - - public void setLoginUserKeytabUsername(String loginUserKeytabUsername) { - this.loginUserKeytabUsername = loginUserKeytabUsername; - } - - public String getLoginUserKeytabPath() { - return loginUserKeytabPath; - } - - public void setLoginUserKeytabPath(String loginUserKeytabPath) { - this.loginUserKeytabPath = loginUserKeytabPath; - } - - public String getJavaSecurityKrb5Conf() { - return javaSecurityKrb5Conf; - } - - public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) { - this.javaSecurityKrb5Conf = javaSecurityKrb5Conf; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceProcessor.java deleted file mode 100644 index e76a088001..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceProcessor.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource; - -import org.apache.dolphinscheduler.common.enums.DbType; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; - -public interface DatasourceProcessor { - - /** - * check datasource param is valid - */ - void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam); - - /** - * create BaseDataSourceParamDTO by connectionJson - * - * @param connectionJson see{@link org.apache.dolphinscheduler.dao.entity.Datasource} - * @return {@link BaseDataSourceParamDTO} - */ - BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson); - - /** - * create datasource connection parameter which will be stored at DataSource - *
- * see {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams} - */ - ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam); - - /** - * deserialize json to datasource connection param - * - * @param connectionJson {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams} - * @return {@link BaseConnectionParam} - */ - ConnectionParam createConnectionParams(String connectionJson); - - /** - * get datasource Driver - */ - String getDatasourceDriver(); - - /** - * get jdbcUrl by connection param, the jdbcUrl is different with ConnectionParam.jdbcUrl, this method will inject - * other to jdbcUrl - * - * @param connectionParam connection param - */ - String getJdbcUrl(ConnectionParam connectionParam); - - /** - * get connection by connectionParam - * - * @param connectionParam connectionParam - * @return {@link Connection} - */ - Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException, IOException; - - /** - * @return {@link DbType} - */ - DbType getDbType(); -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceParamDTO.java deleted file mode 100644 index f4168fce5f..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceParamDTO.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.clickhouse; - -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.enums.DbType; - -public class ClickHouseDatasourceParamDTO extends BaseDataSourceParamDTO { - - @Override - public String toString() { - return "ClickHouseDatasourceParamDTO{" - + "host='" + host + '\'' - + ", port=" + port - + ", database='" + database + '\'' - + ", userName='" + userName + '\'' - + ", password='" + password + '\'' - + ", other='" + other + '\'' - + '}'; - } - - @Override - public DbType getType() { - return DbType.CLICKHOUSE; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceParamDTO.java deleted file mode 100644 index c989472832..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceParamDTO.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.db2; - -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.enums.DbType; - -public class Db2DatasourceParamDTO extends BaseDataSourceParamDTO { - - @Override - public String toString() { - return "Db2DatasourceParamDTO{" - + "name='" + name + '\'' - + ", note='" + note + '\'' - + ", host='" + host + '\'' - + ", port=" + port - + ", database='" + database + '\'' - + ", userName='" + userName + '\'' - + ", password='" + password + '\'' - + ", other='" + other + '\'' - + '}'; - } - - @Override - public DbType getType() { - return DbType.DB2; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveConnectionParam.java deleted file mode 100644 index e982211df7..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveConnectionParam.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.hive; - -import org.apache.dolphinscheduler.common.datasource.BaseHdfsConnectionParam; - -public class HiveConnectionParam extends BaseHdfsConnectionParam { - @Override - public String toString() { - return "HiveConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + ", principal='" + principal + '\'' - + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' - + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' - + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDataSourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDataSourceParamDTO.java deleted file mode 100644 index 816d08889a..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDataSourceParamDTO.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.hive; - -import org.apache.dolphinscheduler.common.datasource.BaseHdfsDatasourceParamDTO; -import org.apache.dolphinscheduler.common.enums.DbType; - -public class HiveDataSourceParamDTO extends BaseHdfsDatasourceParamDTO { - - @Override - public String toString() { - return "HiveDataSourceParamDTO{" - + "host='" + host + '\'' - + ", port=" + port - + ", database='" + database + '\'' - + ", principal='" + principal + '\'' - + ", userName='" + userName + '\'' - + ", password='" + password + '\'' - + ", other='" + other + '\'' - + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' - + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' - + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' - + '}'; - } - - @Override - public DbType getType() { - return DbType.HIVE; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlConnectionParam.java deleted file mode 100644 index 3c5117c2e2..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlConnectionParam.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.mysql; - -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; - -public class MysqlConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "MysqlConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceParamDTO.java deleted file mode 100644 index 3e8f0ed918..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceParamDTO.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.mysql; - -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.enums.DbType; - -public class MysqlDatasourceParamDTO extends BaseDataSourceParamDTO { - - @Override - public String toString() { - return "MysqlDatasourceParamDTO{" - + "name='" + name + '\'' - + ", note='" + note + '\'' - + ", host='" + host + '\'' - + ", port=" + port - + ", database='" + database + '\'' - + ", userName='" + userName + '\'' - + ", password='" + password + '\'' - + ", other='" + other + '\'' - + '}'; - } - - @Override - public DbType getType() { - return DbType.MYSQL; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleConnectionParam.java deleted file mode 100644 index a59f50badb..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleConnectionParam.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.oracle; - -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.common.enums.DbConnectType; - -public class OracleConnectionParam extends BaseConnectionParam { - protected DbConnectType connectType; - - public DbConnectType getConnectType() { - return connectType; - } - - public void setConnectType(DbConnectType connectType) { - this.connectType = connectType; - } - - @Override - public String toString() { - return "OracleConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + ", connectType=" + connectType - + '}'; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceParamDTO.java deleted file mode 100644 index 615c514074..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceParamDTO.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.oracle; - -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.enums.DbConnectType; -import org.apache.dolphinscheduler.common.enums.DbType; - -public class OracleDatasourceParamDTO extends BaseDataSourceParamDTO { - - private DbConnectType connectType; - - public DbConnectType getConnectType() { - return connectType; - } - - public void setConnectType(DbConnectType connectType) { - this.connectType = connectType; - } - - @Override - public String toString() { - return "OracleDatasourceParamDTO{" - + "name='" + name + '\'' - + ", note='" + note + '\'' - + ", host='" + host + '\'' - + ", port=" + port - + ", database='" + database + '\'' - + ", userName='" + userName + '\'' - + ", password='" + password + '\'' - + ", connectType=" + connectType - + ", other='" + other + '\'' - + '}'; - } - - @Override - public DbType getType() { - return DbType.ORACLE; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceParamDTO.java deleted file mode 100644 index 5d81233a7a..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceParamDTO.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.postgresql; - -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.enums.DbType; - -public class PostgreSqlDatasourceParamDTO extends BaseDataSourceParamDTO { - - @Override - public String toString() { - return "PostgreSqlDatasourceParamDTO{" - + "host='" + host + '\'' - + ", port=" + port - + ", database='" + database + '\'' - + ", userName='" + userName + '\'' - + ", password='" + password + '\'' - + ", other='" + other + '\'' - + '}'; - } - - @Override - public DbType getType() { - return DbType.POSTGRESQL; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceParamDTO.java deleted file mode 100644 index 9477362b03..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceParamDTO.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.presto; - -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.enums.DbType; - -public class PrestoDatasourceParamDTO extends BaseDataSourceParamDTO { - - @Override - public String toString() { - return "PrestoDatasourceParamDTO{" - + "name='" + name + '\'' - + ", note='" + note + '\'' - + ", host='" + host + '\'' - + ", port=" + port - + ", database='" + database + '\'' - + ", userName='" + userName + '\'' - + ", password='" + password + '\'' - + ", other='" + other + '\'' - + '}'; - } - - @Override - public DbType getType() { - return DbType.PRESTO; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkConnectionParam.java deleted file mode 100644 index 725d7b8e2b..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkConnectionParam.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.spark; - -import org.apache.dolphinscheduler.common.datasource.BaseHdfsConnectionParam; - -public class SparkConnectionParam extends BaseHdfsConnectionParam { - @Override - public String toString() { - return "SparkConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + ", principal='" + principal + '\'' - + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' - + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' - + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceParamDTO.java deleted file mode 100644 index 784f7d01d9..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceParamDTO.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.spark; - -import org.apache.dolphinscheduler.common.datasource.BaseHdfsDatasourceParamDTO; -import org.apache.dolphinscheduler.common.enums.DbType; - -public class SparkDatasourceParamDTO extends BaseHdfsDatasourceParamDTO { - - @Override - public String toString() { - return "SparkDatasourceParamDTO{" - + "host='" + host + '\'' - + ", port=" + port - + ", database='" + database + '\'' - + ", principal='" + principal + '\'' - + ", userName='" + userName + '\'' - + ", password='" + password + '\'' - + ", other='" + other + '\'' - + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' - + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' - + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' - + '}'; - } - - @Override - public DbType getType() { - return DbType.SPARK; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceParamDTO.java deleted file mode 100644 index 62c757638f..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceParamDTO.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.datasource.sqlserver; - -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.enums.DbType; - -public class SqlServerDatasourceParamDTO extends BaseDataSourceParamDTO { - - @Override - public String toString() { - return "SqlServerDatasourceParamDTO{" - + "name='" + name + '\'' - + ", note='" + note + '\'' - + ", host='" + host + '\'' - + ", port=" + port - + ", database='" + database + '\'' - + ", userName='" + userName + '\'' - + ", password='" + password + '\'' - + ", other='" + other + '\'' - + '}'; - } - - @Override - public DbType getType() { - return DbType.SQLSERVER; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java deleted file mode 100644 index b994afb5f5..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.enums; - -import static java.util.stream.Collectors.toMap; - -import java.util.Arrays; -import java.util.Map; - -import com.baomidou.mybatisplus.annotation.EnumValue; -import com.google.common.base.Functions; - -public enum DbType { - MYSQL(0), - POSTGRESQL(1), - HIVE(2), - SPARK(3), - CLICKHOUSE(4), - ORACLE(5), - SQLSERVER(6), - DB2(7), - PRESTO(8), - H2(9); - - DbType(int code) { - this.code = code; - } - - @EnumValue - private final int code; - - public int getCode() { - return code; - } - - private static final Map DB_TYPE_MAP = - Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); - - public static DbType of(int type) { - if (DB_TYPE_MAP.containsKey(type)) { - return DB_TYPE_MAP.get(type); - } - return null; - } -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index 0933832632..ffed1d13f4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; +import org.apache.dolphinscheduler.spi.enums.ResUploadType; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 120abab525..96519d4249 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.common.utils; import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; +import org.apache.dolphinscheduler.spi.enums.ResUploadType; import org.apache.directory.api.util.Strings; diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 4005a0afaf..3cf79f3783 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -89,3 +89,6 @@ sudo.enable=true # development state development.state=false + +#datasource.plugin.dir config +datasource.plugin.dir=lib/plugin/datasource diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java index 92f0d7bd49..713709030f 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java @@ -17,9 +17,8 @@ package org.apache.dolphinscheduler.common.utils; -import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.spi.utils.PropertyUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import java.net.InetAddress; @@ -28,7 +27,6 @@ import java.net.UnknownHostException; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; @@ -53,41 +51,6 @@ public class CommonUtilsTest { Assert.assertTrue(true); } - @Test - public void getKerberosStartupState() { - boolean kerberosStartupState = CommonUtils.getKerberosStartupState(); - logger.info("kerberos startup state: {}",kerberosStartupState); - Assert.assertFalse(kerberosStartupState); - PowerMockito.mockStatic(PropertyUtils.class); - PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS"); - PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE); - kerberosStartupState = CommonUtils.getKerberosStartupState(); - logger.info("kerberos startup state: {}",kerberosStartupState); - Assert.assertTrue(kerberosStartupState); - - } - - @Test - public void loadKerberosConf() { - try { - PowerMockito.mockStatic(PropertyUtils.class); - PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS"); - PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE); - PowerMockito.when(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)).thenReturn("/opt/krb5.conf"); - PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)).thenReturn("hdfs-mycluster@ESZ.COM"); - PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)).thenReturn("/opt/hdfs.headless.keytab"); - - PowerMockito.mockStatic(UserGroupInformation.class); - boolean result = CommonUtils.loadKerberosConf(new Configuration()); - Assert.assertTrue(result); - - CommonUtils.loadKerberosConf(null, null, null); - - } catch (Exception e) { - Assert.fail("load Kerberos Conf failed"); - } - } - @Test public void getHdfsDataBasePath() { logger.info(HadoopUtils.getHdfsDataBasePath()); @@ -124,41 +87,4 @@ public class CommonUtilsTest { Assert.assertTrue(true); } - @Test - public void encodePassword() { - - PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); - - Assert.assertEquals("",CommonUtils.encodePassword("")); - Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==",CommonUtils.encodePassword("123456")); - Assert.assertEquals("IUAjJCVeJipJVkZCV2xoVFYwQT0=",CommonUtils.encodePassword("!QAZXSW@")); - Assert.assertEquals("IUAjJCVeJipOV1JtWjJWeUtFQT0=",CommonUtils.encodePassword("5dfger(@")); - - PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); - - Assert.assertEquals("",CommonUtils.encodePassword("")); - Assert.assertEquals("123456",CommonUtils.encodePassword("123456")); - Assert.assertEquals("!QAZXSW@",CommonUtils.encodePassword("!QAZXSW@")); - Assert.assertEquals("5dfger(@",CommonUtils.encodePassword("5dfger(@")); - - } - - @Test - public void decodePassword() { - - PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true"); - - Assert.assertEquals("", CommonUtils.decodePassword("")); - Assert.assertEquals("123456", CommonUtils.decodePassword("IUAjJCVeJipNVEl6TkRVMg==")); - Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("IUAjJCVeJipJVkZCV2xoVFYwQT0=")); - Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("IUAjJCVeJipOV1JtWjJWeUtFQT0=")); - - PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false"); - - Assert.assertEquals("", CommonUtils.decodePassword("")); - Assert.assertEquals("123456", CommonUtils.decodePassword("123456")); - Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("!QAZXSW@")); - Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("5dfger(@")); - } - } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java index 53366777f7..b9f7125093 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java @@ -16,20 +16,23 @@ */ package org.apache.dolphinscheduler.dao; -import com.alibaba.druid.pool.DruidDataSource; -import java.sql.Connection; -import java.util.ArrayList; -import java.util.List; -import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.apache.dolphinscheduler.dao.utils.MysqlPerformance; import org.apache.dolphinscheduler.dao.utils.PostgrePerformance; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.alibaba.druid.pool.DruidDataSource; + /** * database state dao diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java index ecd5be0455..b15d0ef7e1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java @@ -16,14 +16,15 @@ */ package org.apache.dolphinscheduler.dao.entity; -import com.fasterxml.jackson.annotation.JsonFormat; -import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import java.util.Date; + import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; - -import java.util.Date; +import com.fasterxml.jackson.annotation.JsonFormat; @TableName("t_ds_datasource") public class DataSource { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java index 318f24535f..1d6ebe0a08 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java @@ -16,12 +16,13 @@ */ package org.apache.dolphinscheduler.dao.entity; -import com.fasterxml.jackson.annotation.JsonFormat; -import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.spi.enums.DbType; import java.util.Date; +import com.fasterxml.jackson.annotation.JsonFormat; + /** * monitor record for database */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java index a6ad6ca8c3..86709f33b9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java @@ -14,15 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.upgrade; -import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.SchemaUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * upgrade manager */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index 560d9a14c7..a6063c5510 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ConditionType; -import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskType; @@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; +import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java index e7e9c70f47..e00f5e7e03 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java @@ -14,20 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.utils; +package org.apache.dolphinscheduler.dao.utils; import static org.apache.dolphinscheduler.dao.MonitorDBDao.VARIABLE_NAME; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.dao.entity.MonitorRecord; +import org.apache.dolphinscheduler.spi.enums.DbType; + import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.Date; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java index b1cdf6f179..6afd9706e6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java @@ -14,17 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.utils; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.dao.entity.MonitorRecord; +import org.apache.dolphinscheduler.spi.enums.DbType; + import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.Date; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java index 00c2dce9a0..301dd9ca42 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java @@ -24,13 +24,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; -import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DatasourceUser; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.spi.enums.DbType; import java.util.Arrays; import java.util.Date; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml new file mode 100644 index 0000000000..0cde60b2bc --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml @@ -0,0 +1,177 @@ + + + + + + dolphinscheduler-datasource-plugin + org.apache.dolphinscheduler + 2.0.0-SNAPSHOT + + 4.0.0 + + dolphinscheduler-datasource-api + + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + commons-io + commons-io + provided + + + + org.slf4j + slf4j-api + provided + + + + com.google.guava + guava + + + + commons-codec + commons-codec + + + + org.apache.commons + commons-collections4 + provided + + + + com.github.oshi + oshi-core + provided + + + + com.alibaba + druid + + + + org.springframework + spring-jdbc + + + + org.apache.hadoop + hadoop-client + + + org.slf4j + slf4j-log4j12 + + + servlet-api + javax.servlet + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + + org.fusesource.leveldbjni + leveldbjni-all + + + org.apache.zookeeper + zookeeper + + + org.apache.hadoop + hadoop-mapreduce-client-shuffle + + + jersey-client + com.sun.jersey + + + jersey-core + com.sun.jersey + + + jaxb-api + javax.xml.bind + + + log4j + log4j + + + provided + + + + junit + junit + test + + + org.mockito + mockito-core + jar + test + + + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + + + org.jacoco + org.jacoco.agent + runtime + test + + + + + dolphinscheduler-datasource-api-${project.version} + + \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java new file mode 100644 index 0000000000..136f285235 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.client; + +import org.apache.dolphinscheduler.plugin.datasource.api.provider.JdbcDataSourceProvider; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; +import org.apache.dolphinscheduler.spi.exception.PluginException; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; + +import com.alibaba.druid.pool.DruidDataSource; +import com.google.common.base.Stopwatch; + +public class CommonDataSourceClient implements DataSourceClient { + + private static final Logger logger = LoggerFactory.getLogger(CommonDataSourceClient.class); + + public static final String COMMON_USER = "root"; + public static final String COMMON_PASSWORD = "123456"; + public static final String COMMON_VALIDATION_QUERY = "select 1"; + + protected final BaseConnectionParam baseConnectionParam; + protected DruidDataSource druidDataSource; + protected JdbcTemplate jdbcTemplate; + + public CommonDataSourceClient(BaseConnectionParam baseConnectionParam) { + this.baseConnectionParam = baseConnectionParam; + preInit(); + checkEnv(baseConnectionParam); + initClient(baseConnectionParam); + checkClient(); + } + + protected void preInit() { + logger.info("preInit in CommonDataSourceClient"); + } + + protected void checkEnv(BaseConnectionParam baseConnectionParam) { + checkValidationQuery(baseConnectionParam); + checkUser(baseConnectionParam); + } + + protected void initClient(BaseConnectionParam baseConnectionParam) { + this.druidDataSource = JdbcDataSourceProvider.createJdbcDataSource(baseConnectionParam); + this.jdbcTemplate = new JdbcTemplate(druidDataSource); + } + + protected void checkUser(BaseConnectionParam baseConnectionParam) { + if (StringUtils.isBlank(baseConnectionParam.getUser())) { + setDefaultUsername(baseConnectionParam); + } + if (StringUtils.isBlank(baseConnectionParam.getPassword())) { + setDefaultPassword(baseConnectionParam); + } + } + + protected void setDefaultUsername(BaseConnectionParam baseConnectionParam) { + baseConnectionParam.setUser(COMMON_USER); + } + + protected void setDefaultPassword(BaseConnectionParam baseConnectionParam) { + baseConnectionParam.setPassword(COMMON_PASSWORD); + } + + protected void checkValidationQuery(BaseConnectionParam baseConnectionParam) { + if (StringUtils.isBlank(baseConnectionParam.getValidationQuery())) { + setDefaultValidationQuery(baseConnectionParam); + } + } + + protected void setDefaultValidationQuery(BaseConnectionParam baseConnectionParam) { + baseConnectionParam.setValidationQuery(COMMON_VALIDATION_QUERY); + } + + @Override + public void checkClient() { + //Checking data source client + Stopwatch stopwatch = Stopwatch.createStarted(); + try { + this.jdbcTemplate.execute(this.baseConnectionParam.getValidationQuery()); + } catch (Exception e) { + throw PluginException.getInstance("JDBC connect failed", e); + } finally { + logger.info("Time to execute check jdbc client with sql {} for {} ms ", this.baseConnectionParam.getValidationQuery(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); + } + } + + @Override + public Connection getConnection() { + try { + return this.druidDataSource.getConnection(); + } catch (SQLException e) { + logger.error("get druidDataSource Connection fail SQLException: {}", e.getMessage(), e); + return null; + } + } + + @Override + public void close() { + logger.info("do close dataSource."); + this.druidDataSource.close(); + this.druidDataSource = null; + this.jdbcTemplate = null; + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDatasourceProcessor.java similarity index 81% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDatasourceProcessor.java index a9d3bcef36..30ac9ead62 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDatasourceProcessor.java @@ -15,10 +15,15 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.collections4.MapUtils; +import java.text.MessageFormat; import java.util.Map; import java.util.regex.Pattern; @@ -76,4 +81,9 @@ public abstract class AbstractDatasourceProcessor implements DatasourceProcessor } } + @Override + public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { + BaseConnectionParam baseConnectionParam = (BaseConnectionParam) connectionParam; + return MessageFormat.format("{0}@{1}@{2}", dbType.getDescp(), baseConnectionParam.getUser(), baseConnectionParam.getJdbcUrl()); + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java similarity index 79% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseDataSourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java index 8bc3f94379..d6d6cab32f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseDataSourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource; - -import org.apache.dolphinscheduler.common.datasource.clickhouse.ClickHouseDatasourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.db2.Db2DatasourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.hive.HiveDataSourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.presto.PrestoDatasourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.spark.SparkDatasourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerDatasourceParamDTO; -import org.apache.dolphinscheduler.common.enums.DbType; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickHouseDatasourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2DatasourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlDatasourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDatasourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSqlDatasourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDatasourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDatasourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SqlServerDatasourceParamDTO; +import org.apache.dolphinscheduler.spi.enums.DbType; import java.io.Serializable; import java.util.Map; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseHdfsConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHdfsConnectionParam.java similarity index 92% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseHdfsConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHdfsConnectionParam.java index ab24027487..b5fc59b511 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseHdfsConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHdfsConnectionParam.java @@ -15,7 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class BaseHdfsConnectionParam extends BaseConnectionParam { protected String principal; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseHdfsDatasourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHdfsDatasourceParamDTO.java similarity index 96% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseHdfsDatasourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHdfsDatasourceParamDTO.java index 0cfa3cb434..aaa1892b88 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseHdfsDatasourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHdfsDatasourceParamDTO.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource; public abstract class BaseHdfsDatasourceParamDTO extends BaseDataSourceParamDTO { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DatasourceProcessor.java similarity index 87% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DatasourceProcessor.java index b0243a4f72..63c434c916 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DatasourceProcessor.java @@ -15,8 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; import java.io.IOException; @@ -30,6 +31,12 @@ public interface DatasourceProcessor { */ void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam); + /** + * get Datasource Client UniqueId + * @return UniqueId + */ + String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType); + /** * create BaseDataSourceParamDTO by connectionJson * @@ -58,6 +65,11 @@ public interface DatasourceProcessor { */ String getDatasourceDriver(); + /** + * get validation Query + */ + String getValidationQuery(); + /** * get jdbcUrl by connection param, the jdbcUrl is different with ConnectionParam.jdbcUrl, this method will inject * other to jdbcUrl diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceParamDTO.java similarity index 89% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceParamDTO.java index 0cdb20fdfe..7938204aac 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceParamDTO.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.clickhouse; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbType; public class ClickHouseDatasourceParamDTO extends BaseDataSourceParamDTO { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceProcessor.java similarity index 80% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceProcessor.java index 81611a3e12..330967b2b8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceProcessor.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.clickhouse; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.ConnectionParam; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang.StringUtils; import java.sql.Connection; import java.sql.DriverManager; @@ -64,8 +64,11 @@ public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor { clickhouseConnectionParam.setAddress(address); clickhouseConnectionParam.setJdbcUrl(jdbcUrl); clickhouseConnectionParam.setUser(clickHouseParam.getUserName()); - clickhouseConnectionParam.setPassword(CommonUtils.encodePassword(clickHouseParam.getPassword())); + clickhouseConnectionParam.setPassword(PasswordUtils.encodePassword(clickHouseParam.getPassword())); + clickhouseConnectionParam.setDriverClassName(getDatasourceDriver()); + clickhouseConnectionParam.setValidationQuery(getValidationQuery()); clickhouseConnectionParam.setOther(transformOther(clickHouseParam.getOther())); + clickhouseConnectionParam.setProps(clickHouseParam.getOther()); return clickhouseConnectionParam; } @@ -79,6 +82,11 @@ public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor { return Constants.COM_CLICKHOUSE_JDBC_DRIVER; } + @Override + public String getValidationQuery() { + return Constants.CLICKHOUSE_VALIDATION_QUERY; + } + @Override public String getJdbcUrl(ConnectionParam connectionParam) { ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; @@ -94,7 +102,7 @@ public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor { ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam), - clickhouseConnectionParam.getUser(), CommonUtils.decodePassword(clickhouseConnectionParam.getPassword())); + clickhouseConnectionParam.getUser(), PasswordUtils.decodePassword(clickhouseConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickhouseConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickhouseConnectionParam.java similarity index 78% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickhouseConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickhouseConnectionParam.java index c01d0005a9..5049cf6253 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickhouseConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickhouseConnectionParam.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.clickhouse; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class ClickhouseConnectionParam extends BaseConnectionParam { @Override @@ -28,6 +28,9 @@ public class ClickhouseConnectionParam extends BaseConnectionParam { + ", address='" + address + '\'' + ", database='" + database + '\'' + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + ", other='" + other + '\'' + '}'; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2ConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2ConnectionParam.java similarity index 78% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2ConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2ConnectionParam.java index ba66c5e16e..c77b3a78ca 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2ConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2ConnectionParam.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.db2; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class Db2ConnectionParam extends BaseConnectionParam { @Override @@ -28,6 +28,9 @@ public class Db2ConnectionParam extends BaseConnectionParam { + ", address='" + address + '\'' + ", database='" + database + '\'' + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + ", other='" + other + '\'' + '}'; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceParamDTO.java similarity index 89% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceParamDTO.java index b5476a525e..a0adfd4fd3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceParamDTO.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.db2; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbType; public class Db2DatasourceParamDTO extends BaseDataSourceParamDTO { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceProcessor.java similarity index 79% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceProcessor.java index 4bad7f6061..d9b6f8d277 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceProcessor.java @@ -15,19 +15,19 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.db2; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.ConnectionParam; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang.StringUtils; import java.sql.Connection; import java.sql.DriverManager; @@ -65,8 +65,11 @@ public class Db2DatasourceProcessor extends AbstractDatasourceProcessor { db2ConnectionParam.setDatabase(db2Param.getDatabase()); db2ConnectionParam.setJdbcUrl(jdbcUrl); db2ConnectionParam.setUser(db2Param.getUserName()); - db2ConnectionParam.setPassword(CommonUtils.encodePassword(db2Param.getPassword())); + db2ConnectionParam.setPassword(PasswordUtils.encodePassword(db2Param.getPassword())); + db2ConnectionParam.setDriverClassName(getDatasourceDriver()); + db2ConnectionParam.setValidationQuery(getValidationQuery()); db2ConnectionParam.setOther(transformOther(db2Param.getOther())); + db2ConnectionParam.setProps(db2Param.getOther()); return db2ConnectionParam; } @@ -95,7 +98,7 @@ public class Db2DatasourceProcessor extends AbstractDatasourceProcessor { Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(db2ConnectionParam), - db2ConnectionParam.getUser(), CommonUtils.decodePassword(db2ConnectionParam.getPassword())); + db2ConnectionParam.getUser(), PasswordUtils.decodePassword(db2ConnectionParam.getPassword())); } @Override @@ -103,6 +106,11 @@ public class Db2DatasourceProcessor extends AbstractDatasourceProcessor { return DbType.DB2; } + @Override + public String getValidationQuery() { + return Constants.DB2_VALIDATION_QUERY; + } + private String transformOther(Map otherMap) { if (MapUtils.isEmpty(otherMap)) { return null; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveConnectionParam.java similarity index 81% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveConnectionParam.java index 006f12917b..0b1ce52349 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveConnectionParam.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.hive; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseHdfsConnectionParam; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHdfsConnectionParam; public class HiveConnectionParam extends BaseHdfsConnectionParam { @Override @@ -28,6 +28,9 @@ public class HiveConnectionParam extends BaseHdfsConnectionParam { + ", address='" + address + '\'' + ", database='" + database + '\'' + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + ", other='" + other + '\'' + ", principal='" + principal + '\'' + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDataSourceParamDTO.java similarity index 90% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDataSourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDataSourceParamDTO.java index 678f866b19..4aaceaae23 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDataSourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDataSourceParamDTO.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.hive; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseHdfsDatasourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHdfsDatasourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbType; public class HiveDataSourceParamDTO extends BaseHdfsDatasourceParamDTO { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDatasourceProcessor.java similarity index 83% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDatasourceProcessor.java index 8113e59e39..2f7c7afd7f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDatasourceProcessor.java @@ -15,20 +15,20 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.hive; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.ConnectionParam; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.HiveConfUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang.StringUtils; import java.io.IOException; import java.sql.Connection; @@ -83,7 +83,9 @@ public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { hiveConnectionParam.setAddress(address.toString()); hiveConnectionParam.setJdbcUrl(jdbcUrl); hiveConnectionParam.setUser(hiveParam.getUserName()); - hiveConnectionParam.setPassword(CommonUtils.encodePassword(hiveParam.getPassword())); + hiveConnectionParam.setPassword(PasswordUtils.encodePassword(hiveParam.getPassword())); + hiveConnectionParam.setDriverClassName(getDatasourceDriver()); + hiveConnectionParam.setValidationQuery(getValidationQuery()); if (CommonUtils.getKerberosStartupState()) { hiveConnectionParam.setPrincipal(hiveParam.getPrincipal()); @@ -92,6 +94,7 @@ public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { hiveConnectionParam.setLoginUserKeytabUsername(hiveParam.getLoginUserKeytabUsername()); } hiveConnectionParam.setOther(transformOther(hiveParam.getOther())); + hiveConnectionParam.setProps(hiveParam.getOther()); return hiveConnectionParam; } @@ -105,6 +108,11 @@ public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; } + @Override + public String getValidationQuery() { + return Constants.HIVE_VALIDATION_QUERY; + } + @Override public String getJdbcUrl(ConnectionParam connectionParam) { HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; @@ -123,7 +131,7 @@ public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { hiveConnectionParam.getLoginUserKeytabUsername(), hiveConnectionParam.getLoginUserKeytabPath()); Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(connectionParam), - hiveConnectionParam.getUser(), CommonUtils.decodePassword(hiveConnectionParam.getPassword())); + hiveConnectionParam.getUser(), PasswordUtils.decodePassword(hiveConnectionParam.getPassword())); } @Override @@ -152,11 +160,7 @@ public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { String[] otherArray = otherParams.split(";", -1); for (String conf : otherArray) { - if (HiveConfUtils.isHiveConfVar(conf)) { - hiveConfListSb.append(conf).append(";"); - } else { - sessionVarListSb.append(conf).append(";"); - } + sessionVarListSb.append(conf).append(";"); } // remove the last ";" diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlConnectionParam.java similarity index 78% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlConnectionParam.java index ea7a574db5..c86af3b915 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlConnectionParam.java @@ -15,12 +15,11 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.mysql; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class MysqlConnectionParam extends BaseConnectionParam { - @Override public String toString() { return "MysqlConnectionParam{" @@ -29,6 +28,9 @@ public class MysqlConnectionParam extends BaseConnectionParam { + ", address='" + address + '\'' + ", database='" + database + '\'' + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + ", other='" + other + '\'' + '}'; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceParamDTO.java similarity index 89% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceParamDTO.java index 3bcb84bd60..25d5a95acd 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceParamDTO.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.mysql; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbType; public class MysqlDatasourceParamDTO extends BaseDataSourceParamDTO { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceProcessor.java similarity index 84% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceProcessor.java index 564ed3dfc2..b84bcc3030 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceProcessor.java @@ -15,19 +15,19 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.mysql; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.ConnectionParam; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang.StringUtils; import java.sql.Connection; import java.sql.DriverManager; @@ -82,8 +82,11 @@ public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor { mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase()); mysqlConnectionParam.setAddress(address); mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName()); - mysqlConnectionParam.setPassword(CommonUtils.encodePassword(mysqlDatasourceParam.getPassword())); + mysqlConnectionParam.setPassword(PasswordUtils.encodePassword(mysqlDatasourceParam.getPassword())); + mysqlConnectionParam.setDriverClassName(getDatasourceDriver()); + mysqlConnectionParam.setValidationQuery(getValidationQuery()); mysqlConnectionParam.setOther(transformOther(mysqlDatasourceParam.getOther())); + mysqlConnectionParam.setProps(mysqlDatasourceParam.getOther()); return mysqlConnectionParam; } @@ -98,6 +101,11 @@ public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor { return Constants.COM_MYSQL_JDBC_DRIVER; } + @Override + public String getValidationQuery() { + return Constants.MYSQL_VALIDATION_QUERY; + } + @Override public String getJdbcUrl(ConnectionParam connectionParam) { MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; @@ -117,7 +125,7 @@ public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor { logger.warn("sensitive param : {} in username field is filtered", AUTO_DESERIALIZE); user = user.replace(AUTO_DESERIALIZE, ""); } - String password = CommonUtils.decodePassword(mysqlConnectionParam.getPassword()); + String password = PasswordUtils.decodePassword(mysqlConnectionParam.getPassword()); if (password.contains(AUTO_DESERIALIZE)) { logger.warn("sensitive param : {} in password field is filtered", AUTO_DESERIALIZE); password = password.replace(AUTO_DESERIALIZE, ""); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleConnectionParam.java similarity index 82% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleConnectionParam.java index 45c7ae14de..9984174042 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleConnectionParam.java @@ -15,13 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.oracle; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbConnectType; public class OracleConnectionParam extends BaseConnectionParam { - protected DbConnectType connectType; public DbConnectType getConnectType() { @@ -40,6 +39,9 @@ public class OracleConnectionParam extends BaseConnectionParam { + ", address='" + address + '\'' + ", database='" + database + '\'' + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + ", other='" + other + '\'' + ", connectType=" + connectType + '}'; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceParamDTO.java similarity index 91% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceParamDTO.java index 502a893579..b027266cf3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceParamDTO.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.oracle; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbConnectType; import org.apache.dolphinscheduler.spi.enums.DbType; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessor.java similarity index 81% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessor.java index 4ec7e19bb6..20a4da6344 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessor.java @@ -15,17 +15,17 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.oracle; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.ConnectionParam; -import org.apache.dolphinscheduler.common.enums.DbConnectType; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbConnectType; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang.StringUtils; @@ -76,12 +76,15 @@ public class OracleDatasourceProcessor extends AbstractDatasourceProcessor { OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); oracleConnectionParam.setUser(oracleParam.getUserName()); - oracleConnectionParam.setPassword(CommonUtils.encodePassword(oracleParam.getPassword())); + oracleConnectionParam.setPassword(PasswordUtils.encodePassword(oracleParam.getPassword())); oracleConnectionParam.setAddress(address); oracleConnectionParam.setJdbcUrl(jdbcUrl); oracleConnectionParam.setDatabase(oracleParam.getDatabase()); oracleConnectionParam.setConnectType(oracleParam.getConnectType()); + oracleConnectionParam.setDriverClassName(getDatasourceDriver()); + oracleConnectionParam.setValidationQuery(getValidationQuery()); oracleConnectionParam.setOther(transformOther(oracleParam.getOther())); + oracleConnectionParam.setProps(oracleParam.getOther()); return oracleConnectionParam; } @@ -96,6 +99,11 @@ public class OracleDatasourceProcessor extends AbstractDatasourceProcessor { return Constants.COM_ORACLE_JDBC_DRIVER; } + @Override + public String getValidationQuery() { + return Constants.ORACLE_VALIDATION_QUERY; + } + @Override public String getJdbcUrl(ConnectionParam connectionParam) { OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; @@ -110,7 +118,7 @@ public class OracleDatasourceProcessor extends AbstractDatasourceProcessor { OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(connectionParam), - oracleConnectionParam.getUser(), CommonUtils.decodePassword(oracleConnectionParam.getPassword())); + oracleConnectionParam.getUser(), PasswordUtils.decodePassword(oracleConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlConnectionParam.java similarity index 78% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlConnectionParam.java index 7090581f65..e2e3dc0a29 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlConnectionParam.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.postgresql; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class PostgreSqlConnectionParam extends BaseConnectionParam { @Override @@ -28,6 +28,9 @@ public class PostgreSqlConnectionParam extends BaseConnectionParam { + ", address='" + address + '\'' + ", database='" + database + '\'' + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + ", other='" + other + '\'' + '}'; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceParamDTO.java similarity index 89% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceParamDTO.java index ef1ec9eda4..b17f8380be 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceParamDTO.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.postgresql; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbType; public class PostgreSqlDatasourceParamDTO extends BaseDataSourceParamDTO { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceProcessor.java similarity index 80% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceProcessor.java index e7939630a4..749c6b5944 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceProcessor.java @@ -15,16 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.postgresql; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.ConnectionParam; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang.StringUtils; @@ -65,8 +65,11 @@ public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor { postgreSqlConnectionParam.setAddress(address); postgreSqlConnectionParam.setDatabase(postgreSqlParam.getDatabase()); postgreSqlConnectionParam.setUser(postgreSqlParam.getUserName()); - postgreSqlConnectionParam.setPassword(CommonUtils.encodePassword(postgreSqlParam.getPassword())); + postgreSqlConnectionParam.setPassword(PasswordUtils.encodePassword(postgreSqlParam.getPassword())); + postgreSqlConnectionParam.setDriverClassName(getDatasourceDriver()); + postgreSqlConnectionParam.setValidationQuery(getValidationQuery()); postgreSqlConnectionParam.setOther(transformOther(postgreSqlParam.getOther())); + postgreSqlConnectionParam.setProps(postgreSqlParam.getOther()); return postgreSqlConnectionParam; } @@ -81,6 +84,11 @@ public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor { return Constants.ORG_POSTGRESQL_DRIVER; } + @Override + public String getValidationQuery() { + return Constants.POSTGRESQL_VALIDATION_QUERY; + } + @Override public String getJdbcUrl(ConnectionParam connectionParam) { PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; @@ -95,7 +103,7 @@ public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor { PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam), - postgreSqlConnectionParam.getUser(), CommonUtils.decodePassword(postgreSqlConnectionParam.getPassword())); + postgreSqlConnectionParam.getUser(), PasswordUtils.decodePassword(postgreSqlConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoConnectionParam.java similarity index 78% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoConnectionParam.java index 436bc6dd1e..fcdd17eaaa 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoConnectionParam.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.presto; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class PrestoConnectionParam extends BaseConnectionParam { @Override @@ -28,6 +28,9 @@ public class PrestoConnectionParam extends BaseConnectionParam { + ", address='" + address + '\'' + ", database='" + database + '\'' + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + ", other='" + other + '\'' + '}'; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceParamDTO.java similarity index 89% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceParamDTO.java index b592e217c2..50e65b5b55 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceParamDTO.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.presto; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbType; public class PrestoDatasourceParamDTO extends BaseDataSourceParamDTO { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceProcessor.java similarity index 80% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceProcessor.java index 25dab29a68..b9fbc33b02 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceProcessor.java @@ -15,16 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.presto; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.ConnectionParam; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang.StringUtils; @@ -64,11 +64,14 @@ public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor { PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam(); prestoConnectionParam.setUser(prestoParam.getUserName()); - prestoConnectionParam.setPassword(CommonUtils.encodePassword(prestoParam.getPassword())); + prestoConnectionParam.setPassword(PasswordUtils.encodePassword(prestoParam.getPassword())); prestoConnectionParam.setOther(transformOther(prestoParam.getOther())); prestoConnectionParam.setAddress(address); prestoConnectionParam.setJdbcUrl(jdbcUrl); prestoConnectionParam.setDatabase(prestoParam.getDatabase()); + prestoConnectionParam.setDriverClassName(getDatasourceDriver()); + prestoConnectionParam.setValidationQuery(getValidationQuery()); + prestoConnectionParam.setProps(prestoParam.getOther()); return prestoConnectionParam; } @@ -83,6 +86,11 @@ public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor { return Constants.COM_PRESTO_JDBC_DRIVER; } + @Override + public String getValidationQuery() { + return Constants.PRESTO_VALIDATION_QUERY; + } + @Override public String getJdbcUrl(ConnectionParam connectionParam) { PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; @@ -97,7 +105,7 @@ public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor { PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(connectionParam), - prestoConnectionParam.getUser(), CommonUtils.decodePassword(prestoConnectionParam.getPassword())); + prestoConnectionParam.getUser(), PasswordUtils.decodePassword(prestoConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkConnectionParam.java similarity index 81% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkConnectionParam.java index fad385e3ce..bd1bb9e025 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkConnectionParam.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.spark; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseHdfsConnectionParam; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHdfsConnectionParam; public class SparkConnectionParam extends BaseHdfsConnectionParam { @Override @@ -28,6 +28,9 @@ public class SparkConnectionParam extends BaseHdfsConnectionParam { + ", address='" + address + '\'' + ", database='" + database + '\'' + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + ", other='" + other + '\'' + ", principal='" + principal + '\'' + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceParamDTO.java similarity index 90% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceParamDTO.java index 144fae5a1f..82e1f1124f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceParamDTO.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.spark; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseHdfsDatasourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHdfsDatasourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbType; public class SparkDatasourceParamDTO extends BaseHdfsDatasourceParamDTO { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceProcessor.java similarity index 82% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceProcessor.java index 7c7d021056..376399d778 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceProcessor.java @@ -15,19 +15,20 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.spark; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.ConnectionParam; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang.StringUtils; import java.io.IOException; import java.sql.Connection; @@ -81,12 +82,16 @@ public class SparkDatasourceProcessor extends AbstractDatasourceProcessor { } SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); - sparkConnectionParam.setPassword(CommonUtils.encodePassword(sparkDatasourceParam.getPassword())); + sparkConnectionParam.setPassword(PasswordUtils.encodePassword(sparkDatasourceParam.getPassword())); sparkConnectionParam.setUser(sparkDatasourceParam.getUserName()); sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther())); sparkConnectionParam.setDatabase(sparkDatasourceParam.getDatabase()); sparkConnectionParam.setAddress(address.toString()); sparkConnectionParam.setJdbcUrl(jdbcUrl); + sparkConnectionParam.setDriverClassName(getDatasourceDriver()); + sparkConnectionParam.setValidationQuery(getValidationQuery()); + sparkConnectionParam.setProps(sparkDatasourceParam.getOther()); + if (CommonUtils.getKerberosStartupState()) { sparkConnectionParam.setPrincipal(sparkDatasourceParam.getPrincipal()); sparkConnectionParam.setJavaSecurityKrb5Conf(sparkDatasourceParam.getJavaSecurityKrb5Conf()); @@ -107,6 +112,11 @@ public class SparkDatasourceProcessor extends AbstractDatasourceProcessor { return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; } + @Override + public String getValidationQuery() { + return Constants.HIVE_VALIDATION_QUERY; + } + @Override public String getJdbcUrl(ConnectionParam connectionParam) { SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; @@ -123,7 +133,7 @@ public class SparkDatasourceProcessor extends AbstractDatasourceProcessor { sparkConnectionParam.getLoginUserKeytabUsername(), sparkConnectionParam.getLoginUserKeytabPath()); Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(sparkConnectionParam), - sparkConnectionParam.getUser(), CommonUtils.decodePassword(sparkConnectionParam.getPassword())); + sparkConnectionParam.getUser(), PasswordUtils.decodePassword(sparkConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerConnectionParam.java similarity index 78% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerConnectionParam.java index ff5225771a..d362c241aa 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerConnectionParam.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.sqlserver; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class SqlServerConnectionParam extends BaseConnectionParam { @Override @@ -28,6 +28,9 @@ public class SqlServerConnectionParam extends BaseConnectionParam { + ", address='" + address + '\'' + ", database='" + database + '\'' + ", jdbcUrl='" + jdbcUrl + '\'' + + ", driverLocation='" + driverLocation + '\'' + + ", driverClassName='" + driverClassName + '\'' + + ", validationQuery='" + validationQuery + '\'' + ", other='" + other + '\'' + '}'; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceParamDTO.java similarity index 89% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceParamDTO.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceParamDTO.java index 101ab2aa9f..b90bb8820b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceParamDTO.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource.sqlserver; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.spi.enums.DbType; public class SqlServerDatasourceParamDTO extends BaseDataSourceParamDTO { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceProcessor.java similarity index 80% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceProcessor.java index 32fca091f1..c4c2d9fec1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceProcessor.java @@ -15,16 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.sqlserver; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.ConnectionParam; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang.StringUtils; @@ -64,7 +64,10 @@ public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor { sqlServerConnectionParam.setJdbcUrl(jdbcUrl); sqlServerConnectionParam.setOther(transformOther(sqlServerParam.getOther())); sqlServerConnectionParam.setUser(sqlServerParam.getUserName()); - sqlServerConnectionParam.setPassword(CommonUtils.encodePassword(sqlServerParam.getPassword())); + sqlServerConnectionParam.setPassword(PasswordUtils.encodePassword(sqlServerParam.getPassword())); + sqlServerConnectionParam.setDriverClassName(getDatasourceDriver()); + sqlServerConnectionParam.setValidationQuery(getValidationQuery()); + sqlServerConnectionParam.setProps(sqlServerParam.getOther()); return sqlServerConnectionParam; } @@ -78,6 +81,11 @@ public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor { return Constants.COM_SQLSERVER_JDBC_DRIVER; } + @Override + public String getValidationQuery() { + return Constants.SQLSERVER_VALIDATION_QUERY; + } + @Override public String getJdbcUrl(ConnectionParam connectionParam) { SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; @@ -93,7 +101,7 @@ public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor { SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(connectionParam), sqlServerConnectionParam.getUser(), - CommonUtils.decodePassword(sqlServerConnectionParam.getPassword())); + PasswordUtils.decodePassword(sqlServerConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java new file mode 100644 index 0000000000..129b30acac --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.plugin; + +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.exception.PluginException; +import org.apache.dolphinscheduler.spi.plugin.DolphinPluginLoader; +import org.apache.dolphinscheduler.spi.plugin.DolphinPluginManagerConfig; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.PropertyUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +public class DataSourceClientProvider { + + private static final Logger logger = LoggerFactory.getLogger(DataSourceClientProvider.class); + + private static final Map uniqueId2dataSourceClientMap = new ConcurrentHashMap<>(); + + private DataSourcePluginManager dataSourcePluginManager; + + private DataSourceClientProvider() { + initDataSourcePlugin(); + } + + private static class DataSourceClientProviderHolder { + private static final DataSourceClientProvider INSTANCE = new DataSourceClientProvider(); + } + + public static DataSourceClientProvider getInstance() { + return DataSourceClientProviderHolder.INSTANCE; + } + + public Connection getConnection(DbType dbType, ConnectionParam connectionParam) { + BaseConnectionParam baseConnectionParam = (BaseConnectionParam) connectionParam; + String datasourceUniqueId = DatasourceUtil.getDatasourceUniqueId(baseConnectionParam, dbType); + logger.info("getConnection datasourceUniqueId {}", datasourceUniqueId); + + DataSourceClient dataSourceClient; + dataSourceClient = uniqueId2dataSourceClientMap.get(datasourceUniqueId); + if (dataSourceClient == null) { + Map dataSourceChannelMap = dataSourcePluginManager.getDataSourceChannelMap(); + DataSourceChannel dataSourceChannel = dataSourceChannelMap.get(dbType.getDescp()); + if (null == dataSourceChannel) { + throw PluginException.getInstance(String.format("%s dataSource Plugin Not Found,Please Check Config File.", dbType.getDescp())); + } + dataSourceClient = dataSourceChannel.createDataSourceClient(baseConnectionParam); + uniqueId2dataSourceClientMap.put(datasourceUniqueId, dataSourceClient); + } + return dataSourceClient.getConnection(); + } + + private void initDataSourcePlugin() { + dataSourcePluginManager = new DataSourcePluginManager(); + DolphinPluginManagerConfig datasourcePluginManagerConfig = new DolphinPluginManagerConfig(); + datasourcePluginManagerConfig.setPlugins(PropertyUtils.getString(Constants.DATASOURCE_PLUGIN_BINDING)); + + datasourcePluginManagerConfig.setInstalledPluginsDir(PropertyUtils.getString(Constants.DATASOURCE_PLUGIN_DIR, Constants.DATASOURCE_PLUGIN_PATH)); + + if (StringUtils.isNotBlank(PropertyUtils.getString(Constants.MAVEN_LOCAL_REPOSITORY))) { + datasourcePluginManagerConfig.setMavenLocalRepository(PropertyUtils.getString(Constants.MAVEN_LOCAL_REPOSITORY).trim()); + } + + DolphinPluginLoader datasourcePluginLoader = new DolphinPluginLoader(datasourcePluginManagerConfig, ImmutableList.of(dataSourcePluginManager)); + try { + datasourcePluginLoader.loadPlugins(); + } catch (Exception e) { + throw PluginException.getInstance("Load datasource Plugin Failed !", e); + } + if (MapUtils.isEmpty(dataSourcePluginManager.getDataSourceChannelMap())) { + throw PluginException.getInstance("datasource Plugin Not Found,Please Check Config File"); + } + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java new file mode 100644 index 0000000000..01b92dd2cd --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.plugin; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +import static com.google.common.base.Preconditions.checkState; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.classloader.ThreadContextClassLoader; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.exception.PluginException; +import org.apache.dolphinscheduler.spi.plugin.AbstractDolphinPluginManager; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DataSourcePluginManager extends AbstractDolphinPluginManager { + + private static final Logger logger = LoggerFactory.getLogger(DataSourcePluginManager.class); + + private final Map datasourceClientFactoryMap = new ConcurrentHashMap<>(); + private final Map datasourceClientMap = new ConcurrentHashMap<>(); + + public Map getDataSourceChannelMap() { + return datasourceClientMap; + } + + private void addDatasourceClientFactory(DataSourceChannelFactory datasourceChannelFactory) { + requireNonNull(datasourceChannelFactory, "datasourceChannelFactory is null"); + + if (datasourceClientFactoryMap.putIfAbsent(datasourceChannelFactory.getName(), datasourceChannelFactory) != null) { + throw PluginException.getInstance(format("Datasource Plugin '%s' is already registered", datasourceChannelFactory.getName())); + } + + try { + loadDatasourceClient(datasourceChannelFactory.getName()); + } catch (Exception e) { + throw PluginException.getInstance(format("Datasource Plugin '%s' is can not load .", datasourceChannelFactory.getName())); + } + } + + private void loadDatasourceClient(String name) { + requireNonNull(name, "name is null"); + + DataSourceChannelFactory datasourceChannelFactory = datasourceClientFactoryMap.get(name); + checkState(datasourceChannelFactory != null, "datasource Plugin {} is not registered", name); + + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(datasourceChannelFactory.getClass().getClassLoader())) { + DataSourceChannel datasourceChannel = datasourceChannelFactory.create(); + this.datasourceClientMap.put(name, datasourceChannel); + } + + logger.info("-- Loaded datasource Plugin {} --", name); + } + + @Override + public void installPlugin(DolphinSchedulerPlugin dolphinSchedulerPlugin) { + for (DataSourceChannelFactory datasourceChannelFactory : dolphinSchedulerPlugin.getDatasourceChannelFactorys()) { + logger.info("Registering datasource Plugin '{}'", datasourceChannelFactory.getName()); + this.addDatasourceClientFactory(datasourceChannelFactory); + } + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JdbcDataSourceProvider.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JdbcDataSourceProvider.java new file mode 100644 index 0000000000..d84db4478e --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JdbcDataSourceProvider.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.provider; + +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.PropertyUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.druid.pool.DruidDataSource; + +/** + * Jdbc Data Source Provider + */ +public class JdbcDataSourceProvider { + + private static final Logger logger = LoggerFactory.getLogger(JdbcDataSourceProvider.class); + + public static DruidDataSource createJdbcDataSource(BaseConnectionParam properties) { + logger.info("Creating DruidDataSource pool for maxActive:{}", PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50)); + + DruidDataSource druidDataSource = new DruidDataSource(); + + druidDataSource.setDriverClassName(properties.getDriverClassName()); + druidDataSource.setUrl(properties.getJdbcUrl()); + druidDataSource.setUsername(properties.getUser()); + druidDataSource.setPassword(PasswordUtils.decodePassword(properties.getPassword())); + + druidDataSource.setMinIdle(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MIN_IDLE, 5)); + druidDataSource.setMaxActive(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50)); + druidDataSource.setTestOnBorrow(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_TEST_ON_BORROW, false)); + + if (properties.getProps() != null) { + properties.getProps().forEach(druidDataSource::addConnectionProperty); + } + + logger.info("Creating DruidDataSource pool success."); + return druidDataSource; + } + + /** + * @return One Session Jdbc DataSource + */ + public static DruidDataSource createOneSessionJdbcDataSource(BaseConnectionParam properties) { + logger.info("Creating OneSession DruidDataSource pool for maxActive:{}", PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50)); + + DruidDataSource druidDataSource = new DruidDataSource(); + + druidDataSource.setDriverClassName(properties.getDriverClassName()); + druidDataSource.setUrl(properties.getJdbcUrl()); + druidDataSource.setUsername(properties.getUser()); + druidDataSource.setPassword(PasswordUtils.decodePassword(properties.getPassword())); + + druidDataSource.setMinIdle(1); + druidDataSource.setMaxActive(1); + druidDataSource.setTestOnBorrow(true); + + if (properties.getProps() != null) { + properties.getProps().forEach(druidDataSource::addConnectionProperty); + } + + logger.info("Creating OneSession DruidDataSource pool success."); + return druidDataSource; + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/util/CommonUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java similarity index 96% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/util/CommonUtils.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java index 0c771389bd..25b6a59d46 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/util/CommonUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.util; +package org.apache.dolphinscheduler.plugin.datasource.api.utils; import static org.apache.dolphinscheduler.spi.task.TaskConstants.HADOOP_SECURITY_AUTHENTICATION; import static org.apache.dolphinscheduler.spi.task.TaskConstants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE; @@ -41,8 +41,6 @@ import java.io.IOException; */ public class CommonUtils { - public static final String resourceUploadPath = PropertyUtils.getString(TaskConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler"); - private CommonUtils() { throw new UnsupportedOperationException("Construct CommonUtils"); } @@ -130,6 +128,7 @@ public class CommonUtils { * @return data hdfs path */ public static String getHdfsDataBasePath() { + String resourceUploadPath = PropertyUtils.getString(TaskConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler"); if ("/".equals(resourceUploadPath)) { // if basepath is configured to /, the generated url may be //default/resources (with extra leading /) return ""; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtil.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DatasourceUtil.java similarity index 73% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtil.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DatasourceUtil.java index 45dcece4ac..19dd7a7555 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtil.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DatasourceUtil.java @@ -15,20 +15,21 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource; - -import org.apache.dolphinscheduler.common.datasource.clickhouse.ClickHouseDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.db2.Db2DatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.hive.HiveDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.presto.PrestoDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.spark.SparkDatasourceProcessor; -import org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerDatasourceProcessor; -import org.apache.dolphinscheduler.common.enums.DbType; - -import java.sql.Connection; +package org.apache.dolphinscheduler.plugin.datasource.api.utils; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickHouseDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2DatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSqlDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SqlServerDatasourceProcessor; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,14 +78,6 @@ public class DatasourceUtil { return getDatasourceProcessor(dbType).createConnectionParams(connectionJson); } - public static Connection getConnection(DbType dbType, ConnectionParam connectionParam) { - try { - return getDatasourceProcessor(dbType).getConnection(connectionParam); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - public static String getJdbcUrl(DbType dbType, ConnectionParam baseConnectionParam) { return getDatasourceProcessor(dbType).getJdbcUrl(baseConnectionParam); } @@ -118,4 +111,10 @@ public class DatasourceUtil { } } + /** + * get datasource UniqueId + */ + public static String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { + return getDatasourceProcessor(dbType).getDatasourceUniqueId(connectionParam, dbType); + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/PasswordUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/PasswordUtils.java similarity index 98% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/PasswordUtils.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/PasswordUtils.java index d71b7d04ee..9bf2f96735 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/PasswordUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/PasswordUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource; +package org.apache.dolphinscheduler.plugin.datasource.api.utils; import static org.apache.dolphinscheduler.spi.task.TaskConstants.DATASOURCE_ENCRYPTION_ENABLE; import static org.apache.dolphinscheduler.spi.task.TaskConstants.DATASOURCE_ENCRYPTION_SALT; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClientTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClientTest.java new file mode 100644 index 0000000000..7bb5a3c8e5 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClientTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.client; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlConnectionParam; +import org.apache.dolphinscheduler.plugin.datasource.api.provider.JdbcDataSourceProvider; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +import java.sql.Connection; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; +import org.powermock.modules.junit4.PowerMockRunner; +import org.springframework.jdbc.core.JdbcTemplate; + +import com.alibaba.druid.pool.DruidDataSource; + +@RunWith(PowerMockRunner.class) +@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient") +@PrepareForTest(value = {DruidDataSource.class, CommonDataSourceClient.class, JdbcDataSourceProvider.class, JdbcTemplate.class, Connection.class}) +public class CommonDataSourceClientTest { + + @Mock + private CommonDataSourceClient commonDataSourceClient; + + @Test + public void testPreInit() { + PowerMockito.doNothing().when(commonDataSourceClient).preInit(); + commonDataSourceClient.preInit(); + Mockito.verify(commonDataSourceClient).preInit(); + } + + @Test + public void testCheckEnv() { + BaseConnectionParam baseConnectionParam = new MysqlConnectionParam(); + PowerMockito.doNothing().when(commonDataSourceClient).checkEnv(Mockito.any(BaseConnectionParam.class)); + commonDataSourceClient.checkEnv(baseConnectionParam); + Mockito.verify(commonDataSourceClient).checkEnv(Mockito.any(BaseConnectionParam.class)); + + PowerMockito.doNothing().when(commonDataSourceClient).checkValidationQuery(Mockito.any(BaseConnectionParam.class)); + commonDataSourceClient.checkValidationQuery(baseConnectionParam); + Mockito.verify(commonDataSourceClient).checkValidationQuery(Mockito.any(BaseConnectionParam.class)); + + PowerMockito.doNothing().when(commonDataSourceClient).checkUser(Mockito.any(BaseConnectionParam.class)); + commonDataSourceClient.checkUser(baseConnectionParam); + Mockito.verify(commonDataSourceClient).checkUser(Mockito.any(BaseConnectionParam.class)); + + PowerMockito.doNothing().when(commonDataSourceClient).setDefaultUsername(Mockito.any(BaseConnectionParam.class)); + commonDataSourceClient.setDefaultUsername(baseConnectionParam); + Mockito.verify(commonDataSourceClient).setDefaultUsername(Mockito.any(BaseConnectionParam.class)); + + PowerMockito.doNothing().when(commonDataSourceClient).setDefaultPassword(Mockito.any(BaseConnectionParam.class)); + commonDataSourceClient.setDefaultPassword(baseConnectionParam); + Mockito.verify(commonDataSourceClient).setDefaultPassword(Mockito.any(BaseConnectionParam.class)); + + } + + @Test + public void testInitClient() { + BaseConnectionParam baseConnectionParam = new MysqlConnectionParam(); + PowerMockito.doNothing().when(commonDataSourceClient).initClient(Mockito.any(BaseConnectionParam.class)); + commonDataSourceClient.initClient(baseConnectionParam); + Mockito.verify(commonDataSourceClient).initClient(Mockito.any(BaseConnectionParam.class)); + } + + @Test + public void testCheckClient() { + PowerMockito.doNothing().when(this.commonDataSourceClient).checkClient(); + this.commonDataSourceClient.checkClient(); + Mockito.verify(this.commonDataSourceClient).checkClient(); + } + + @Test + public void testGetConnection() { + Connection connection = PowerMockito.mock(Connection.class); + PowerMockito.when(commonDataSourceClient.getConnection()).thenReturn(connection); + Assert.assertNotNull(commonDataSourceClient.getConnection()); + } +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java similarity index 74% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java index 75c1f3c17d..106aa300a1 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java @@ -15,33 +15,46 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.clickhouse; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({Class.class, DriverManager.class}) +@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) public class ClickHouseDatasourceProcessorTest { private ClickHouseDatasourceProcessor clickHouseDatasourceProcessor = new ClickHouseDatasourceProcessor(); @Test public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); ClickHouseDatasourceParamDTO clickhouseConnectionParam = new ClickHouseDatasourceParamDTO(); clickhouseConnectionParam.setUserName("user"); clickhouseConnectionParam.setPassword("password"); clickhouseConnectionParam.setHost("localhost"); clickhouseConnectionParam.setPort(8123); clickhouseConnectionParam.setDatabase("default"); + clickhouseConnectionParam.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) clickHouseDatasourceProcessor .createConnectionParams(clickhouseConnectionParam); Assert.assertNotNull(connectionParams); @@ -80,4 +93,9 @@ public class ClickHouseDatasourceProcessorTest { public void testGetDbType() { Assert.assertEquals(DbType.CLICKHOUSE, clickHouseDatasourceProcessor.getDbType()); } + + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.CLICKHOUSE_VALIDATION_QUERY, clickHouseDatasourceProcessor.getValidationQuery()); + } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceProcessorTest.java similarity index 72% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessorTest.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceProcessorTest.java index 21b1bc4663..7b405aae93 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceProcessorTest.java @@ -15,33 +15,46 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.db2; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({Class.class, DriverManager.class}) +@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) public class Db2DatasourceProcessorTest { private Db2DatasourceProcessor db2DatasourceProcessor = new Db2DatasourceProcessor(); @Test public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO(); db2DatasourceParamDTO.setUserName("root"); db2DatasourceParamDTO.setPassword("123456"); db2DatasourceParamDTO.setHost("localhost"); db2DatasourceParamDTO.setPort(5142); db2DatasourceParamDTO.setDatabase("default"); + db2DatasourceParamDTO.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); Db2ConnectionParam connectionParams = (Db2ConnectionParam) db2DatasourceProcessor .createConnectionParams(db2DatasourceParamDTO); @@ -80,4 +93,8 @@ public class Db2DatasourceProcessorTest { Assert.assertEquals(DbType.DB2, db2DatasourceProcessor.getDbType()); } + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.DB2_VALIDATION_QUERY, db2DatasourceProcessor.getValidationQuery()); + } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDatasourceProcessorTest.java similarity index 68% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessorTest.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDatasourceProcessorTest.java index 230b0855e5..89ca40966f 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDatasourceProcessorTest.java @@ -15,33 +15,48 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.hive; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({Class.class, DriverManager.class}) +@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) public class HiveDatasourceProcessorTest { private HiveDatasourceProcessor hiveDatasourceProcessor = new HiveDatasourceProcessor(); @Test public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); hiveDataSourceParamDTO.setHost("localhost1,localhost2"); hiveDataSourceParamDTO.setPort(5142); hiveDataSourceParamDTO.setUserName("default"); hiveDataSourceParamDTO.setDatabase("default"); + hiveDataSourceParamDTO.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); + PowerMockito.mockStatic(CommonUtils.class); + PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(false); HiveConnectionParam connectionParams = (HiveConnectionParam) hiveDatasourceProcessor .createConnectionParams(hiveDataSourceParamDTO); System.out.println(JSONUtils.toJsonString(connectionParams)); @@ -76,4 +91,9 @@ public class HiveDatasourceProcessorTest { public void testGetDbType() { Assert.assertEquals(DbType.HIVE, hiveDatasourceProcessor.getDbType()); } + + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.HIVE_VALIDATION_QUERY, hiveDatasourceProcessor.getValidationQuery()); + } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceProcessorTest.java similarity index 65% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessorTest.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceProcessorTest.java index 89bf4b15f1..eef74f4bcf 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceProcessorTest.java @@ -15,35 +15,47 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.mysql; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({Class.class, DriverManager.class}) +@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) public class MysqlDatasourceProcessorTest { private MysqlDatasourceProcessor mysqlDatasourceProcessor = new MysqlDatasourceProcessor(); @Test public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); mysqlDatasourceParamDTO.setUserName("root"); mysqlDatasourceParamDTO.setPassword("123456"); mysqlDatasourceParamDTO.setHost("localhost"); mysqlDatasourceParamDTO.setPort(3306); mysqlDatasourceParamDTO.setDatabase("default"); - + mysqlDatasourceParamDTO.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); MysqlConnectionParam connectionParams = (MysqlConnectionParam) mysqlDatasourceProcessor .createConnectionParams(mysqlDatasourceParamDTO); System.out.println(JSONUtils.toJsonString(connectionParams)); @@ -78,4 +90,17 @@ public class MysqlDatasourceProcessorTest { public void testGetDbType() { Assert.assertEquals(DbType.MYSQL, mysqlDatasourceProcessor.getDbType()); } + + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.MYSQL_VALIDATION_QUERY, mysqlDatasourceProcessor.getValidationQuery()); + } + + @Test + public void testGetDatasourceUniqueId() { + MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); + mysqlConnectionParam.setJdbcUrl("jdbc:mysql://localhost:3306/default"); + mysqlConnectionParam.setUser("root"); + Assert.assertEquals("mysql@root@jdbc:mysql://localhost:3306/default", mysqlDatasourceProcessor.getDatasourceUniqueId(mysqlConnectionParam, DbType.MYSQL)); + } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessorTest.java similarity index 72% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessorTest.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessorTest.java index 4f0fce2d9b..de88708810 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessorTest.java @@ -15,28 +15,38 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.oracle; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbConnectType; -import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbConnectType; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({Class.class, DriverManager.class}) +@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) public class OracleDatasourceProcessorTest { private OracleDatasourceProcessor oracleDatasourceProcessor = new OracleDatasourceProcessor(); @Test public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO(); oracleDatasourceParamDTO.setConnectType(DbConnectType.ORACLE_SID); oracleDatasourceParamDTO.setHost("localhost"); @@ -44,7 +54,9 @@ public class OracleDatasourceProcessorTest { oracleDatasourceParamDTO.setUserName("root"); oracleDatasourceParamDTO.setPassword("123456"); oracleDatasourceParamDTO.setDatabase("default"); - + oracleDatasourceParamDTO.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); OracleConnectionParam connectionParams = (OracleConnectionParam) oracleDatasourceProcessor .createConnectionParams(oracleDatasourceParamDTO); Assert.assertNotNull(connectionParams); @@ -80,4 +92,9 @@ public class OracleDatasourceProcessorTest { public void getDbType() { Assert.assertEquals(DbType.ORACLE, oracleDatasourceProcessor.getDbType()); } + + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.ORACLE_VALIDATION_QUERY, oracleDatasourceProcessor.getValidationQuery()); + } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java similarity index 73% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java index 178c30ce7b..0555cbfa13 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java @@ -15,34 +15,46 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.postgresql; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({Class.class, DriverManager.class}) +@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) public class PostgreSqlDatasourceProcessorTest { private PostgreSqlDatasourceProcessor postgreSqlDatasourceProcessor = new PostgreSqlDatasourceProcessor(); @Test public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO(); postgreSqlDatasourceParamDTO.setUserName("root"); postgreSqlDatasourceParamDTO.setPassword("123456"); postgreSqlDatasourceParamDTO.setHost("localhost"); postgreSqlDatasourceParamDTO.setPort(3308); postgreSqlDatasourceParamDTO.setDatabase("default"); - + postgreSqlDatasourceParamDTO.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) postgreSqlDatasourceProcessor .createConnectionParams(postgreSqlDatasourceParamDTO); Assert.assertEquals("jdbc:postgresql://localhost:3308", connectionParams.getAddress()); @@ -80,4 +92,9 @@ public class PostgreSqlDatasourceProcessorTest { public void testGetDbType() { Assert.assertEquals(DbType.POSTGRESQL, postgreSqlDatasourceProcessor.getDbType()); } + + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.POSTGRESQL_VALIDATION_QUERY, postgreSqlDatasourceProcessor.getValidationQuery()); + } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceProcessorTest.java similarity index 72% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessorTest.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceProcessorTest.java index a2bfe997bd..5145cd2e21 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceProcessorTest.java @@ -15,34 +15,46 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.presto; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({Class.class, DriverManager.class}) +@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) public class PrestoDatasourceProcessorTest { private PrestoDatasourceProcessor prestoDatasourceProcessor = new PrestoDatasourceProcessor(); @Test public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO(); prestoDatasourceParamDTO.setHost("localhost"); prestoDatasourceParamDTO.setPort(1234); prestoDatasourceParamDTO.setDatabase("default"); prestoDatasourceParamDTO.setUserName("root"); prestoDatasourceParamDTO.setPassword("123456"); - + prestoDatasourceParamDTO.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); PrestoConnectionParam connectionParams = (PrestoConnectionParam) prestoDatasourceProcessor .createConnectionParams(prestoDatasourceParamDTO); Assert.assertEquals("jdbc:presto://localhost:1234", connectionParams.getAddress()); @@ -78,4 +90,9 @@ public class PrestoDatasourceProcessorTest { public void testGetDbType() { Assert.assertEquals(DbType.PRESTO, prestoDatasourceProcessor.getDbType()); } + + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.PRESTO_VALIDATION_QUERY, prestoDatasourceProcessor.getValidationQuery()); + } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceProcessorTest.java similarity index 70% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessorTest.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceProcessorTest.java index 706c0aef0f..7e55e26848 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceProcessorTest.java @@ -15,34 +15,48 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.spark; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({Class.class, DriverManager.class}) +@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) public class SparkDatasourceProcessorTest { private SparkDatasourceProcessor sparkDatasourceProcessor = new SparkDatasourceProcessor(); @Test public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO(); sparkDatasourceParamDTO.setUserName("root"); sparkDatasourceParamDTO.setPassword("12345"); sparkDatasourceParamDTO.setHost("localhost1,localhost2"); sparkDatasourceParamDTO.setPort(1234); sparkDatasourceParamDTO.setDatabase("default"); - + sparkDatasourceParamDTO.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); + PowerMockito.mockStatic(CommonUtils.class); + PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(false); SparkConnectionParam connectionParams = (SparkConnectionParam) sparkDatasourceProcessor .createConnectionParams(sparkDatasourceParamDTO); Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234", connectionParams.getAddress()); @@ -77,4 +91,9 @@ public class SparkDatasourceProcessorTest { public void testGetDbType() { Assert.assertEquals(DbType.SPARK, sparkDatasourceProcessor.getDbType()); } + + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.HIVE_VALIDATION_QUERY, sparkDatasourceProcessor.getValidationQuery()); + } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceProcessorTest.java similarity index 72% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessorTest.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceProcessorTest.java index 39a23f6e25..de5956035c 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceProcessorTest.java @@ -15,35 +15,47 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.sqlserver; +package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({Class.class, DriverManager.class}) +@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class}) public class SqlServerDatasourceProcessorTest { private SqlServerDatasourceProcessor sqlServerDatasourceProcessor = new SqlServerDatasourceProcessor(); @Test public void testCreateConnectionParams() { + Map props = new HashMap<>(); + props.put("serverTimezone", "utc"); SqlServerDatasourceParamDTO sqlServerDatasourceParamDTO = new SqlServerDatasourceParamDTO(); sqlServerDatasourceParamDTO.setUserName("root"); sqlServerDatasourceParamDTO.setPassword("123456"); sqlServerDatasourceParamDTO.setDatabase("default"); sqlServerDatasourceParamDTO.setHost("localhost"); sqlServerDatasourceParamDTO.setPort(1234); - + sqlServerDatasourceParamDTO.setOther(props); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test"); SqlServerConnectionParam connectionParams = (SqlServerConnectionParam) sqlServerDatasourceProcessor .createConnectionParams(sqlServerDatasourceParamDTO); Assert.assertEquals("jdbc:sqlserver://localhost:1234", connectionParams.getAddress()); @@ -78,4 +90,9 @@ public class SqlServerDatasourceProcessorTest { public void testGetDbType() { Assert.assertEquals(DbType.SQLSERVER, sqlServerDatasourceProcessor.getDbType()); } + + @Test + public void testGetValidationQuery() { + Assert.assertEquals(Constants.SQLSERVER_VALIDATION_QUERY, sqlServerDatasourceProcessor.getValidationQuery()); + } } \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JdbcDataSourceProviderTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JdbcDataSourceProviderTest.java new file mode 100644 index 0000000000..02df95a362 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JdbcDataSourceProviderTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.provider; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlConnectionParam; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.alibaba.druid.pool.DruidDataSource; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(value = {DruidDataSource.class, JdbcDataSourceProvider.class}) +public class JdbcDataSourceProviderTest { + + @Test + public void testCreateJdbcDataSource() { + PowerMockito.mockStatic(JdbcDataSourceProvider.class); + DruidDataSource druidDataSource = PowerMockito.mock(DruidDataSource.class); + PowerMockito.when(JdbcDataSourceProvider.createJdbcDataSource(Mockito.any())).thenReturn(druidDataSource); + Assert.assertNotNull(JdbcDataSourceProvider.createJdbcDataSource(new MysqlConnectionParam())); + } + + @Test + public void testCreateOneSessionJdbcDataSource() { + PowerMockito.mockStatic(JdbcDataSourceProvider.class); + DruidDataSource druidDataSource = PowerMockito.mock(DruidDataSource.class); + PowerMockito.when(JdbcDataSourceProvider.createOneSessionJdbcDataSource(Mockito.any())).thenReturn(druidDataSource); + Assert.assertNotNull(JdbcDataSourceProvider.createOneSessionJdbcDataSource(new MysqlConnectionParam())); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtilsTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtilsTest.java new file mode 100644 index 0000000000..61f78ba634 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtilsTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.utils; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DATASOURCE_ENCRYPTION_ENABLE; + +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.PropertyUtils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@SuppressStaticInitializationFor("org.apache.dolphinscheduler.spi.utils.PropertyUtils") +@PrepareForTest(value = {PropertyUtils.class, UserGroupInformation.class, CommonUtils.class, PasswordUtils.class}) +public class CommonUtilsTest { + + @Test + public void testGetKerberosStartupState() { + PowerMockito.mockStatic(CommonUtils.class); + PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(false); + boolean kerberosStartupState = CommonUtils.getKerberosStartupState(); + Assert.assertFalse(kerberosStartupState); + + PowerMockito.mockStatic(PropertyUtils.class); + PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS"); + PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, true)).thenReturn(Boolean.TRUE); + kerberosStartupState = CommonUtils.getKerberosStartupState(); + Assert.assertFalse(kerberosStartupState); + } + + @Test + public void testLoadKerberosConf() { + try { + PowerMockito.mockStatic(PropertyUtils.class); + PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS"); + PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE); + PowerMockito.when(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)).thenReturn("/opt/krb5.conf"); + PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)).thenReturn("hdfs-mycluster@ESZ.COM"); + PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)).thenReturn("/opt/hdfs.headless.keytab"); + + PowerMockito.mockStatic(UserGroupInformation.class); + boolean result = CommonUtils.loadKerberosConf(new Configuration()); + Assert.assertTrue(result); + + CommonUtils.loadKerberosConf(null, null, null); + + } catch (Exception e) { + Assert.fail("load Kerberos Conf failed"); + } + } + + @Test + public void encodePassword() { + PowerMockito.mockStatic(PropertyUtils.class); + PowerMockito.when(PropertyUtils.getBoolean(DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.TRUE); + + Assert.assertEquals("", PasswordUtils.encodePassword("")); + Assert.assertEquals("bnVsbE1USXpORFUy", PasswordUtils.encodePassword("123456")); + Assert.assertEquals("bnVsbElWRkJXbGhUVjBBPQ==", PasswordUtils.encodePassword("!QAZXSW@")); + Assert.assertEquals("bnVsbE5XUm1aMlZ5S0VBPQ==", PasswordUtils.encodePassword("5dfger(@")); + + PowerMockito.when(PropertyUtils.getBoolean(DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.FALSE); + + Assert.assertEquals("", PasswordUtils.encodePassword("")); + Assert.assertEquals("123456", PasswordUtils.encodePassword("123456")); + Assert.assertEquals("!QAZXSW@", PasswordUtils.encodePassword("!QAZXSW@")); + Assert.assertEquals("5dfger(@", PasswordUtils.encodePassword("5dfger(@")); + + } + + @Test + public void decodePassword() { + PowerMockito.mockStatic(PropertyUtils.class); + PowerMockito.when(PropertyUtils.getBoolean(DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.TRUE); + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true"); + + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.decodePassword("bnVsbE1USXpORFUy")).thenReturn("123456"); + PowerMockito.when(PasswordUtils.decodePassword("bnVsbElWRkJXbGhUVjBBPQ==")).thenReturn("!QAZXSW@"); + PowerMockito.when(PasswordUtils.decodePassword("bnVsbE5XUm1aMlZ5S0VBPQ==")).thenReturn("5dfger(@"); + + Assert.assertEquals(null, PasswordUtils.decodePassword("")); + Assert.assertEquals("123456", PasswordUtils.decodePassword("bnVsbE1USXpORFUy")); + Assert.assertEquals("!QAZXSW@", PasswordUtils.decodePassword("bnVsbElWRkJXbGhUVjBBPQ==")); + Assert.assertEquals("5dfger(@", PasswordUtils.decodePassword("bnVsbE5XUm1aMlZ5S0VBPQ==")); + + PowerMockito.when(PropertyUtils.getBoolean(DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.FALSE); + + PowerMockito.when(PasswordUtils.decodePassword("123456")).thenReturn("123456"); + PowerMockito.when(PasswordUtils.decodePassword("!QAZXSW@")).thenReturn("!QAZXSW@"); + PowerMockito.when(PasswordUtils.decodePassword("5dfger(@")).thenReturn("5dfger(@"); + + Assert.assertEquals(null, PasswordUtils.decodePassword("")); + Assert.assertEquals("123456", PasswordUtils.decodePassword("123456")); + Assert.assertEquals("!QAZXSW@", PasswordUtils.decodePassword("!QAZXSW@")); + Assert.assertEquals("5dfger(@", PasswordUtils.decodePassword("5dfger(@")); + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DatasourceUtilTest.java similarity index 76% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DatasourceUtilTest.java index 1b8b59cd99..0078fb03d5 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DatasourceUtilTest.java @@ -15,19 +15,20 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource; +package org.apache.dolphinscheduler.plugin.datasource.api.utils; -import java.util.HashMap; -import java.util.Map; -import org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam; -import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO; -import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlConnectionParam; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlDatasourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlDatasourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; import org.junit.Test; @@ -38,7 +39,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({Class.class, DriverManager.class, MysqlDatasourceProcessor.class}) +@PrepareForTest({Class.class, DriverManager.class, MysqlDatasourceProcessor.class, DataSourceClientProvider.class, PasswordUtils.class, CommonUtils.class}) public class DatasourceUtilTest { @Test @@ -63,6 +64,10 @@ public class DatasourceUtilTest { mysqlDatasourceParamDTO.setUserName("root"); mysqlDatasourceParamDTO.setPort(3306); mysqlDatasourceParamDTO.setPassword("123456"); + PowerMockito.mockStatic(PasswordUtils.class); + PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("123456"); + PowerMockito.mockStatic(CommonUtils.class); + PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(false); ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO); Assert.assertNotNull(connectionParam); } @@ -80,18 +85,21 @@ public class DatasourceUtilTest { } @Test - public void testGetConnection() throws ClassNotFoundException, SQLException { - PowerMockito.mockStatic(Class.class); - PowerMockito.when(Class.forName(Mockito.any())).thenReturn(null); - PowerMockito.mockStatic(DriverManager.class); - PowerMockito.when(DriverManager.getConnection(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(null); + public void testGetConnection() { + + PowerMockito.mockStatic(DataSourceClientProvider.class); + DataSourceClientProvider clientProvider = PowerMockito.mock(DataSourceClientProvider.class); + PowerMockito.when(DataSourceClientProvider.getInstance()).thenReturn(clientProvider); + + Connection connection = PowerMockito.mock(Connection.class); + PowerMockito.when(clientProvider.getConnection(Mockito.any(), Mockito.any())).thenReturn(connection); MysqlConnectionParam connectionParam = new MysqlConnectionParam(); connectionParam.setUser("root"); connectionParam.setPassword("123456"); - Connection connection = DatasourceUtil.getConnection(DbType.MYSQL, connectionParam); + connection = DataSourceClientProvider.getInstance().getConnection(DbType.MYSQL, connectionParam); - Assert.assertNull(connection); + Assert.assertNotNull(connection); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml new file mode 100644 index 0000000000..316091e459 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml @@ -0,0 +1,114 @@ + + + + + dolphinscheduler-datasource-plugin + org.apache.dolphinscheduler + 2.0.0-SNAPSHOT + + 4.0.0 + + dolphinscheduler-datasource-clickhouse + ${project.artifactId} + + dolphinscheduler-plugin + + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + + + + ru.yandex.clickhouse + clickhouse-jdbc + + + jaxb-api + javax.xml.bind + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + ${clickhouse.jdbc.version} + + + + junit + junit + test + + + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + + + org.mockito + mockito-core + jar + test + + + + org.jacoco + org.jacoco.agent + runtime + test + + + + + + dolphinscheduler-datasource-clickhouse-${project.version} + + \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2ConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannel.java similarity index 59% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2ConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannel.java index bb47ec2aea..7557febdd3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2ConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannel.java @@ -15,20 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.db2; +package org.apache.dolphinscheduler.plugin.datasource.clickhouse; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; + +public class ClickhouseDataSourceChannel implements DataSourceChannel { -public class Db2ConnectionParam extends BaseConnectionParam { @Override - public String toString() { - return "Db2ConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam) { + return new ClickhouseDataSourceClient(baseConnectionParam); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelFactory.java new file mode 100644 index 0000000000..531b71d7c4 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.clickhouse; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +public class ClickhouseDataSourceChannelFactory implements DataSourceChannelFactory { + @Override + public String getName() { + return "clickhouse"; + } + + @Override + public DataSourceChannel create() { + return new ClickhouseDataSourceChannel(); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceClient.java new file mode 100644 index 0000000000..f16599c50c --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceClient.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.clickhouse; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +public class ClickhouseDataSourceClient extends CommonDataSourceClient { + + public ClickhouseDataSourceClient(BaseConnectionParam baseConnectionParam) { + super(baseConnectionParam); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourcePlugin.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourcePlugin.java new file mode 100644 index 0000000000..759f820699 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourcePlugin.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.clickhouse; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class ClickhouseDataSourcePlugin implements DolphinSchedulerPlugin { + @Override + public Iterable getDatasourceChannelFactorys() { + return ImmutableList.of(new ClickhouseDataSourceChannelFactory()); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelFactoryTest.java new file mode 100644 index 0000000000..5bfb884d97 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelFactoryTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.clickhouse; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; + +import org.junit.Assert; +import org.junit.Test; + +public class ClickhouseDataSourceChannelFactoryTest { + + @Test + public void testCreate() { + ClickhouseDataSourceChannelFactory sourceChannelFactory = new ClickhouseDataSourceChannelFactory(); + DataSourceChannel dataSourceChannel = sourceChannelFactory.create(); + Assert.assertNotNull(dataSourceChannel); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelTest.java new file mode 100644 index 0000000000..e7a77b62c2 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.clickhouse; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickhouseConnectionParam; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient") +@PrepareForTest({ClickhouseDataSourceClient.class, ClickhouseDataSourceChannel.class}) +public class ClickhouseDataSourceChannelTest { + + @Test + public void testCreateDataSourceClient() { + ClickhouseDataSourceChannel sourceChannel = PowerMockito.mock(ClickhouseDataSourceChannel.class); + ClickhouseDataSourceClient dataSourceClient = PowerMockito.mock(ClickhouseDataSourceClient.class); + PowerMockito.when(sourceChannel.createDataSourceClient(Mockito.any())).thenReturn(dataSourceClient); + Assert.assertNotNull(sourceChannel.createDataSourceClient(new ClickhouseConnectionParam())); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourcePluginTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourcePluginTest.java new file mode 100644 index 0000000000..a8563a3b44 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourcePluginTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.clickhouse; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import org.junit.Assert; +import org.junit.Test; + +public class ClickhouseDataSourcePluginTest { + + private ClickhouseDataSourcePlugin dataSourcePlugin = new ClickhouseDataSourcePlugin(); + + @Test + public void testGetDatasourceChannelFactorys() { + Iterable channelFactorys = dataSourcePlugin.getDatasourceChannelFactorys(); + for (DataSourceChannelFactory dataSourceChannelFactory : channelFactorys) { + Assert.assertTrue(dataSourceChannelFactory instanceof ClickhouseDataSourceChannelFactory); + } + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/pom.xml new file mode 100644 index 0000000000..2d664c9741 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/pom.xml @@ -0,0 +1,91 @@ + + + + + dolphinscheduler-datasource-plugin + org.apache.dolphinscheduler + 2.0.0-SNAPSHOT + + 4.0.0 + + dolphinscheduler-datasource-db2 + ${project.artifactId} + + dolphinscheduler-plugin + + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + + + + junit + junit + test + + + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + + + org.mockito + mockito-core + jar + test + + + + org.jacoco + org.jacoco.agent + runtime + test + + + + + + dolphinscheduler-datasource-db2-${project.version} + + + \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannel.java new file mode 100644 index 0000000000..92da758ff8 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.db2; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; + +public class DB2DataSourceChannel implements DataSourceChannel { + + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam) { + return new DB2DataSourceClient(baseConnectionParam); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java new file mode 100644 index 0000000000..44a26a3b05 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactory.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.db2; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +public class DB2DataSourceChannelFactory implements DataSourceChannelFactory { + + @Override + public String getName() { + return "db2"; + } + + @Override + public DataSourceChannel create() { + return new DB2DataSourceChannel(); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceClient.java new file mode 100644 index 0000000000..ab59b3f1d5 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceClient.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.db2; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +public class DB2DataSourceClient extends CommonDataSourceClient { + + public DB2DataSourceClient(BaseConnectionParam baseConnectionParam) { + super(baseConnectionParam); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourcePlugin.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourcePlugin.java new file mode 100644 index 0000000000..2ec5ec7281 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourcePlugin.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.db2; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class DB2DataSourcePlugin implements DolphinSchedulerPlugin { + + @Override + public Iterable getDatasourceChannelFactorys() { + return ImmutableList.of(new DB2DataSourceChannelFactory()); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactoryTest.java new file mode 100644 index 0000000000..89be3f70a6 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelFactoryTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.db2; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; + +import org.junit.Assert; +import org.junit.Test; + +public class DB2DataSourceChannelFactoryTest { + + @Test + public void testCreate() { + DB2DataSourceChannelFactory sourceChannelFactory = new DB2DataSourceChannelFactory(); + DataSourceChannel dataSourceChannel = sourceChannelFactory.create(); + Assert.assertNotNull(dataSourceChannel); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelTest.java new file mode 100644 index 0000000000..ea25bb950c --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.db2; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2ConnectionParam; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient") +@PrepareForTest({DB2DataSourceClient.class, DB2DataSourceChannel.class}) +public class DB2DataSourceChannelTest { + + @Test + public void testCreateDataSourceClient() { + DB2DataSourceChannel sourceChannel = PowerMockito.mock(DB2DataSourceChannel.class); + DB2DataSourceClient dataSourceClient = PowerMockito.mock(DB2DataSourceClient.class); + PowerMockito.when(sourceChannel.createDataSourceClient(Mockito.any())).thenReturn(dataSourceClient); + Assert.assertNotNull(sourceChannel.createDataSourceClient(new Db2ConnectionParam())); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourcePluginTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourcePluginTest.java new file mode 100644 index 0000000000..76678f7c53 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourcePluginTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.db2; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import org.junit.Assert; +import org.junit.Test; + +public class DB2DataSourcePluginTest { + + private DB2DataSourcePlugin dataSourcePlugin = new DB2DataSourcePlugin(); + + @Test + public void testGetDatasourceChannelFactorys() { + Iterable channelFactorys = dataSourcePlugin.getDatasourceChannelFactorys(); + for (DataSourceChannelFactory dataSourceChannelFactory : channelFactorys) { + Assert.assertTrue(dataSourceChannelFactory instanceof DataSourceChannelFactory); + } + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml new file mode 100644 index 0000000000..208b3429b9 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml @@ -0,0 +1,371 @@ + + + + + dolphinscheduler-datasource-plugin + org.apache.dolphinscheduler + 2.0.0-SNAPSHOT + + 4.0.0 + + dolphinscheduler-datasource-hive + ${project.artifactId} + + dolphinscheduler-plugin + + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + org.apache.hadoop + hadoop-client + + + org.slf4j + slf4j-log4j12 + + + servlet-api + javax.servlet + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + + org.fusesource.leveldbjni + leveldbjni-all + + + org.apache.zookeeper + zookeeper + + + org.apache.hadoop + hadoop-mapreduce-client-shuffle + + + jersey-client + com.sun.jersey + + + jersey-core + com.sun.jersey + + + jaxb-api + javax.xml.bind + + + log4j + log4j + + + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + + + + + org.apache.hive + hive-jdbc + ${hive.jdbc.version} + + + slf4j-log4j12 + org.slf4j + + + org.eclipse.jetty.aggregate + jetty-all + + + + org.apache.ant + ant + + + io.dropwizard.metrics + metrics-json + + + io.dropwizard.metrics + metrics-jvm + + + com.github.joshelser + dropwizard-metrics-hadoop-metrics2-reporter + + + + io.netty + netty-all + + + com.google.code.gson + gson + + + com.google.code.findbugs + jsr305 + + + io.dropwizard.metrics + metrics-core + + + javax.servlet + servlet-api + + + org.apache.avro + avro + + + org.apache.commons + commons-compress + + + org.apache.curator + curator-client + + + org.apache.hadoop + hadoop-auth + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.apache.hadoop + hadoop-yarn-api + + + + org.apache.zookeeper + zookeeper + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + com.google.protobuf + protobuf-java + + + org.json + json + + + log4j-slf4j-impl + org.apache.logging.log4j + + + javax.servlet + org.eclipse.jetty.orbit + + + servlet-api-2.5 + org.mortbay.jetty + + + jasper-runtime + tomcat + + + slider-core + org.apache.slider + + + hbase-server + org.apache.hbase + + + jersey-client + com.sun.jersey + + + jersey-core + com.sun.jersey + + + jersey-json + com.sun.jersey + + + jersey-server + com.sun.jersey + + + jersey-guice + com.sun.jersey.contribs + + + hbase-common + org.apache.hbase + + + hbase-hadoop2-compat + org.apache.hbase + + + hbase-client + org.apache.hbase + + + hbase-hadoop-compat + org.apache.hbase + + + tephra-hbase-compat-1.0 + co.cask.tephra + + + jaxb-api + javax.xml.bind + + + hive-llap-client + org.apache.hive + + + hive-llap-common + org.apache.hive + + + hive-llap-server + org.apache.hive + + + tephra-core + co.cask.tephra + + + ant + ant + + + stringtemplate + org.antlr + + + antlr-runtime + org.antlr + + + hive-shims + org.apache.hive + + + jsp-api + javax.servlet + + + log4j-api + org.apache.logging.log4j + + + log4j-core + org.apache.logging.log4j + + + log4j-web + org.apache.logging.log4j + + + jasper-compiler + tomcat + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + + + + junit + junit + test + + + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + + + org.mockito + mockito-core + jar + test + + + + org.jacoco + org.jacoco.agent + runtime + test + + + + + + dolphinscheduler-datasource-hive-${project.version} + + + \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannel.java new file mode 100644 index 0000000000..106fe99f51 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.hive; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; + +public class HiveDataSourceChannel implements DataSourceChannel { + + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam) { + return new HiveDataSourceClient(baseConnectionParam); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java new file mode 100644 index 0000000000..93e98f1501 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.hive; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +public class HiveDataSourceChannelFactory implements DataSourceChannelFactory { + @Override + public String getName() { + return "hive"; + } + + @Override + public DataSourceChannel create() { + return new HiveDataSourceChannel(); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceClient.java new file mode 100644 index 0000000000..7784fb6eeb --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceClient.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.hive; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JAVA_SECURITY_KRB5_CONF; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JAVA_SECURITY_KRB5_CONF_PATH; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.plugin.datasource.api.provider.JdbcDataSourceProvider; +import org.apache.dolphinscheduler.plugin.datasource.utils.CommonUtil; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.exception.PluginException; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.PropertyUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; + +import com.alibaba.druid.pool.DruidDataSource; + +public class HiveDataSourceClient extends CommonDataSourceClient { + + private static final Logger logger = LoggerFactory.getLogger(HiveDataSourceClient.class); + + protected DruidDataSource oneSessionDataSource; + private JdbcTemplate oneSessionJdbcTemplate; + private UserGroupInformation ugi; + + public HiveDataSourceClient(BaseConnectionParam baseConnectionParam) { + super(baseConnectionParam); + } + + @Override + protected void initClient(BaseConnectionParam baseConnectionParam) { + logger.info("Create UserGroupInformation."); + this.ugi = createUserGroupInformation(baseConnectionParam.getUser()); + logger.info("Create ugi success."); + + super.initClient(baseConnectionParam); + this.oneSessionDataSource = JdbcDataSourceProvider.createOneSessionJdbcDataSource(baseConnectionParam); + this.oneSessionJdbcTemplate = new JdbcTemplate(oneSessionDataSource); + logger.info("Init {} success.", getClass().getName()); + } + + @Override + protected void checkEnv(BaseConnectionParam baseConnectionParam) { + super.checkEnv(baseConnectionParam); + checkKerberosEnv(); + } + + private void checkKerberosEnv() { + String krb5File = PropertyUtils.getString(JAVA_SECURITY_KRB5_CONF_PATH); + if (StringUtils.isNotBlank(krb5File)) { + System.setProperty(JAVA_SECURITY_KRB5_CONF, krb5File); + } + } + + private UserGroupInformation createUserGroupInformation(String username) { + String krb5File = PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH); + String keytab = PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH); + String principal = PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME); + try { + return CommonUtil.createUGI(getHadoopConf(), principal, keytab, krb5File, username); + } catch (IOException e) { + throw PluginException.getInstance("createUserGroupInformation fail. ", e); + } + } + + protected Configuration getHadoopConf() { + return new Configuration(); + } + + @Override + public Connection getConnection() { + try { + return oneSessionDataSource.getConnection(); + } catch (SQLException e) { + logger.error("get oneSessionDataSource Connection fail SQLException: {}", e.getMessage(), e); + return null; + } + } + + @Override + public void close() { + super.close(); + logger.info("close HiveDataSourceClient."); + + this.oneSessionDataSource.close(); + this.oneSessionDataSource = null; + this.oneSessionJdbcTemplate = null; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourcePlugin.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourcePlugin.java new file mode 100644 index 0000000000..06f32f0c3e --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourcePlugin.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.hive; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class HiveDataSourcePlugin implements DolphinSchedulerPlugin { + @Override + public Iterable getDatasourceChannelFactorys() { + return ImmutableList.of(new HiveDataSourceChannelFactory()); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/utils/CommonUtil.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/utils/CommonUtil.java new file mode 100644 index 0000000000..faf8f4aafa --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/utils/CommonUtil.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.utils; + +import static org.apache.dolphinscheduler.spi.utils.Constants.JAVA_SECURITY_KRB5_CONF; + +import org.apache.dolphinscheduler.spi.enums.ResUploadType; +import org.apache.dolphinscheduler.spi.utils.Constants; +import org.apache.dolphinscheduler.spi.utils.PropertyUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.IOException; +import java.util.Objects; + +public class CommonUtil { + + private CommonUtil() { + } + + public static boolean getKerberosStartupState() { + String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); + return resUploadType == ResUploadType.HDFS && kerberosStartupState; + } + + public static synchronized UserGroupInformation createUGI(Configuration configuration, String principal, String keyTab, String krb5File, String username) + throws IOException { + if (getKerberosStartupState()) { + Objects.requireNonNull(keyTab); + if (StringUtils.isNotBlank(krb5File)) { + System.setProperty(JAVA_SECURITY_KRB5_CONF, krb5File); + } + return loginKerberos(configuration, principal, keyTab); + } + return UserGroupInformation.createRemoteUser(username); + } + + public static synchronized UserGroupInformation loginKerberos(final Configuration config, final String principal, final String keyTab) + throws IOException { + config.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS); + UserGroupInformation.setConfiguration(config); + UserGroupInformation.loginUserFromKeytab(principal.trim(), keyTab.trim()); + return UserGroupInformation.getCurrentUser(); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactoryTest.java new file mode 100644 index 0000000000..6f74a3dd03 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactoryTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.hive; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; + +import org.junit.Assert; +import org.junit.Test; + +public class HiveDataSourceChannelFactoryTest { + + @Test + public void testCreate() { + HiveDataSourceChannelFactory sourceChannelFactory = new HiveDataSourceChannelFactory(); + DataSourceChannel dataSourceChannel = sourceChannelFactory.create(); + Assert.assertNotNull(dataSourceChannel); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelTest.java new file mode 100644 index 0000000000..b19d44e0a6 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.hive; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveConnectionParam; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient") +@PrepareForTest({HiveDataSourceChannel.class, HiveDataSourceClient.class}) +public class HiveDataSourceChannelTest { + + @Test + public void testCreateDataSourceClient() { + HiveDataSourceChannel sourceChannel = PowerMockito.mock(HiveDataSourceChannel.class); + HiveDataSourceClient dataSourceClient = PowerMockito.mock(HiveDataSourceClient.class); + PowerMockito.when(sourceChannel.createDataSourceClient(Mockito.any())).thenReturn(dataSourceClient); + Assert.assertNotNull(sourceChannel.createDataSourceClient(new HiveConnectionParam())); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourcePluginTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourcePluginTest.java new file mode 100644 index 0000000000..0ca4015d8f --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourcePluginTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.hive; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import org.junit.Assert; +import org.junit.Test; + +public class HiveDataSourcePluginTest { + + private HiveDataSourcePlugin dataSourcePlugin = new HiveDataSourcePlugin(); + + @Test + public void testGetDatasourceChannelFactorys() { + Iterable channelFactorys = dataSourcePlugin.getDatasourceChannelFactorys(); + for (DataSourceChannelFactory dataSourceChannelFactory : channelFactorys) { + Assert.assertTrue(dataSourceChannelFactory instanceof DataSourceChannelFactory); + } + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/pom.xml new file mode 100644 index 0000000000..0197dc4107 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/pom.xml @@ -0,0 +1,95 @@ + + + + + dolphinscheduler-datasource-plugin + org.apache.dolphinscheduler + 2.0.0-SNAPSHOT + + 4.0.0 + + dolphinscheduler-datasource-mysql + ${project.artifactId} + + dolphinscheduler-plugin + + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + + + + mysql + mysql-connector-java + + + + junit + junit + test + + + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + + + org.mockito + mockito-core + jar + test + + + + org.jacoco + org.jacoco.agent + runtime + test + + + + + + dolphinscheduler-datasource-mysql-${project.version} + + \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannel.java new file mode 100644 index 0000000000..6764403c10 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.mysql; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; + +public class MysqlDataSourceChannel implements DataSourceChannel { + + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam) { + return new MysqlDataSourceClient(baseConnectionParam); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannelFactory.java new file mode 100644 index 0000000000..ee379db27e --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannelFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.mysql; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +public class MysqlDataSourceChannelFactory implements DataSourceChannelFactory { + @Override + public String getName() { + return "mysql"; + } + + @Override + public DataSourceChannel create() { + return new MysqlDataSourceChannel(); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceClient.java new file mode 100644 index 0000000000..2e2a7dd735 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceClient.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.mysql; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +public class MysqlDataSourceClient extends CommonDataSourceClient { + + public MysqlDataSourceClient(BaseConnectionParam baseConnectionParam) { + super(baseConnectionParam); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourcePlugin.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourcePlugin.java new file mode 100644 index 0000000000..145f7db0fe --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourcePlugin.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.mysql; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class MysqlDataSourcePlugin implements DolphinSchedulerPlugin { + @Override + public Iterable getDatasourceChannelFactorys() { + return ImmutableList.of(new MysqlDataSourceChannelFactory()); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannelFactoryTest.java new file mode 100644 index 0000000000..dd0603f8f7 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannelFactoryTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.mysql; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; + +import org.junit.Assert; +import org.junit.Test; + +public class MysqlDataSourceChannelFactoryTest { + + @Test + public void testCreate() { + MysqlDataSourceChannelFactory sourceChannelFactory = new MysqlDataSourceChannelFactory(); + DataSourceChannel dataSourceChannel = sourceChannelFactory.create(); + Assert.assertNotNull(dataSourceChannel); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannelTest.java new file mode 100644 index 0000000000..216485c7b9 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourceChannelTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.mysql; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlConnectionParam; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient") +@PrepareForTest({MysqlDataSourceClient.class, MysqlDataSourceChannel.class}) +public class MysqlDataSourceChannelTest { + + @Test + public void testCreateDataSourceClient() { + MysqlDataSourceChannel sourceChannel = PowerMockito.mock(MysqlDataSourceChannel.class); + MysqlDataSourceClient dataSourceClient = PowerMockito.mock(MysqlDataSourceClient.class); + PowerMockito.when(sourceChannel.createDataSourceClient(Mockito.any())).thenReturn(dataSourceClient); + Assert.assertNotNull(sourceChannel.createDataSourceClient(new MysqlConnectionParam())); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourcePluginTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourcePluginTest.java new file mode 100644 index 0000000000..415342bbb4 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MysqlDataSourcePluginTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.mysql; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import org.junit.Assert; +import org.junit.Test; + +public class MysqlDataSourcePluginTest { + + private MysqlDataSourcePlugin dataSourcePlugin = new MysqlDataSourcePlugin(); + + @Test + public void testGetDatasourceChannelFactorys() { + Iterable channelFactorys = dataSourcePlugin.getDatasourceChannelFactorys(); + for (DataSourceChannelFactory dataSourceChannelFactory : channelFactorys) { + Assert.assertTrue(dataSourceChannelFactory instanceof DataSourceChannelFactory); + } + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/pom.xml new file mode 100644 index 0000000000..04a2e28100 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/pom.xml @@ -0,0 +1,91 @@ + + + + + dolphinscheduler-datasource-plugin + org.apache.dolphinscheduler + 2.0.0-SNAPSHOT + + 4.0.0 + + dolphinscheduler-datasource-oracle + ${project.artifactId} + + dolphinscheduler-plugin + + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + + + + junit + junit + test + + + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + + + org.mockito + mockito-core + jar + test + + + + org.jacoco + org.jacoco.agent + runtime + test + + + + + + dolphinscheduler-datasource-oracle-${project.version} + + + \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannel.java new file mode 100644 index 0000000000..7dd06b8952 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.oracle; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; + +public class OracleDataSourceChannel implements DataSourceChannel { + + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam) { + return new OracleDataSourceClient(baseConnectionParam); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java new file mode 100644 index 0000000000..9fbfdf1e5e --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.oracle; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +public class OracleDataSourceChannelFactory implements DataSourceChannelFactory { + @Override + public String getName() { + return "oracle"; + } + + @Override + public DataSourceChannel create() { + return new OracleDataSourceChannel(); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceClient.java new file mode 100644 index 0000000000..8706022de7 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceClient.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.oracle; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +public class OracleDataSourceClient extends CommonDataSourceClient { + + public OracleDataSourceClient(BaseConnectionParam baseConnectionParam) { + super(baseConnectionParam); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourcePlugin.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourcePlugin.java new file mode 100644 index 0000000000..67ce6e25c2 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourcePlugin.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.oracle; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class OracleDataSourcePlugin implements DolphinSchedulerPlugin { + @Override + public Iterable getDatasourceChannelFactorys() { + return ImmutableList.of(new OracleDataSourceChannelFactory()); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactoryTest.java new file mode 100644 index 0000000000..0dc4af4635 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactoryTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.oracle; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; + +import org.junit.Assert; +import org.junit.Test; + +public class OracleDataSourceChannelFactoryTest { + + @Test + public void testCreate() { + OracleDataSourceChannelFactory sourceChannelFactory = new OracleDataSourceChannelFactory(); + DataSourceChannel dataSourceChannel = sourceChannelFactory.create(); + Assert.assertNotNull(dataSourceChannel); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelTest.java new file mode 100644 index 0000000000..82d836bd8e --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.oracle; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleConnectionParam; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient") +@PrepareForTest({OracleDataSourceClient.class, OracleDataSourceChannel.class}) +public class OracleDataSourceChannelTest { + + @Test + public void testCreateDataSourceClient() { + OracleDataSourceChannel sourceChannel = PowerMockito.mock(OracleDataSourceChannel.class); + OracleDataSourceClient dataSourceClient = PowerMockito.mock(OracleDataSourceClient.class); + PowerMockito.when(sourceChannel.createDataSourceClient(Mockito.any())).thenReturn(dataSourceClient); + Assert.assertNotNull(sourceChannel.createDataSourceClient(new OracleConnectionParam())); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourcePluginTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourcePluginTest.java new file mode 100644 index 0000000000..c6bec874d0 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourcePluginTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.oracle; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import org.junit.Assert; +import org.junit.Test; + +public class OracleDataSourcePluginTest { + + private OracleDataSourcePlugin dataSourcePlugin = new OracleDataSourcePlugin(); + + @Test + public void testGetDatasourceChannelFactorys() { + Iterable channelFactorys = dataSourcePlugin.getDatasourceChannelFactorys(); + for (DataSourceChannelFactory dataSourceChannelFactory : channelFactorys) { + Assert.assertTrue(dataSourceChannelFactory instanceof DataSourceChannelFactory); + } + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml new file mode 100644 index 0000000000..fed48005a7 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml @@ -0,0 +1,100 @@ + + + + + dolphinscheduler-datasource-plugin + org.apache.dolphinscheduler + 2.0.0-SNAPSHOT + + 4.0.0 + + dolphinscheduler-datasource-postgresql + ${project.artifactId} + + dolphinscheduler-plugin + + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + + + + com.google.guava + guava + + + + org.postgresql + postgresql + + + + junit + junit + test + + + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + + + org.mockito + mockito-core + jar + test + + + + org.jacoco + org.jacoco.agent + runtime + test + + + + + + dolphinscheduler-datasource-postgresql-${project.version} + + \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannel.java new file mode 100644 index 0000000000..ce2ab2248f --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.postgresql; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; + +public class PostgresqlDataSourceChannel implements DataSourceChannel { + + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam) { + return new PostgresqlDataSourceClient(baseConnectionParam); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannelFactory.java new file mode 100644 index 0000000000..2dc001ee23 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannelFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.postgresql; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +public class PostgresqlDataSourceChannelFactory implements DataSourceChannelFactory { + @Override + public String getName() { + return "postgresql"; + } + + @Override + public DataSourceChannel create() { + return new PostgresqlDataSourceChannel(); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceClient.java new file mode 100644 index 0000000000..19e8b53ef5 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceClient.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.postgresql; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +public class PostgresqlDataSourceClient extends CommonDataSourceClient { + + public PostgresqlDataSourceClient(BaseConnectionParam baseConnectionParam) { + super(baseConnectionParam); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourcePlugin.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourcePlugin.java new file mode 100644 index 0000000000..6b2ce27e97 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourcePlugin.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.postgresql; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class PostgresqlDataSourcePlugin implements DolphinSchedulerPlugin { + @Override + public Iterable getDatasourceChannelFactorys() { + return ImmutableList.of(new PostgresqlDataSourceChannelFactory()); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannelFactoryTest.java new file mode 100644 index 0000000000..7b45926778 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannelFactoryTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.postgresql; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; + +import org.junit.Assert; +import org.junit.Test; + +public class PostgresqlDataSourceChannelFactoryTest { + + @Test + public void testCreate() { + PostgresqlDataSourceChannelFactory sourceChannelFactory = new PostgresqlDataSourceChannelFactory(); + DataSourceChannel dataSourceChannel = sourceChannelFactory.create(); + Assert.assertNotNull(dataSourceChannel); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannelTest.java new file mode 100644 index 0000000000..1251540cd1 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourceChannelTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.postgresql; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSqlConnectionParam; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient") +@PrepareForTest({PostgresqlDataSourceClient.class, PostgresqlDataSourceChannel.class}) +public class PostgresqlDataSourceChannelTest { + + @Test + public void testCreateDataSourceClient() { + PostgresqlDataSourceChannel sourceChannel = PowerMockito.mock(PostgresqlDataSourceChannel.class); + PostgresqlDataSourceClient dataSourceClient = PowerMockito.mock(PostgresqlDataSourceClient.class); + PowerMockito.when(sourceChannel.createDataSourceClient(Mockito.any())).thenReturn(dataSourceClient); + Assert.assertNotNull(sourceChannel.createDataSourceClient(new PostgreSqlConnectionParam())); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickhouseConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourcePluginTest.java similarity index 54% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickhouseConnectionParam.java rename to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourcePluginTest.java index d2fdd0a333..b91299f3a2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickhouseConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgresqlDataSourcePluginTest.java @@ -15,20 +15,23 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource.clickhouse; +package org.apache.dolphinscheduler.plugin.datasource.postgresql; -import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; -public class ClickhouseConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "ClickhouseConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; +import org.junit.Assert; +import org.junit.Test; + +public class PostgresqlDataSourcePluginTest { + + private PostgresqlDataSourcePlugin dataSourcePlugin = new PostgresqlDataSourcePlugin(); + + @Test + public void testGetDatasourceChannelFactorys() { + Iterable channelFactorys = dataSourcePlugin.getDatasourceChannelFactorys(); + for (DataSourceChannelFactory dataSourceChannelFactory : channelFactorys) { + Assert.assertTrue(dataSourceChannelFactory instanceof DataSourceChannelFactory); + } } + } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/pom.xml new file mode 100644 index 0000000000..c0dbde8c9a --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/pom.xml @@ -0,0 +1,102 @@ + + + + + + dolphinscheduler-datasource-plugin + org.apache.dolphinscheduler + 2.0.0-SNAPSHOT + + 4.0.0 + + dolphinscheduler-datasource-sqlserver + ${project.artifactId} + + dolphinscheduler-plugin + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + + + + com.microsoft.sqlserver + mssql-jdbc + + + azure-keyvault + com.microsoft.azure + + + ${mssql.jdbc.version} + + + + junit + junit + test + + + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + + + org.mockito + mockito-core + jar + test + + + + org.jacoco + org.jacoco.agent + runtime + test + + + + + + dolphinscheduler-datasource-sqlserver-${project.version} + + + \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlServerDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlServerDataSourceChannel.java new file mode 100644 index 0000000000..ce9cce713a --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlServerDataSourceChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sqlserver; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; + +public class SqlServerDataSourceChannel implements DataSourceChannel { + + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam) { + return new SqlserverDataSourceClient(baseConnectionParam); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlServerDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlServerDataSourceChannelFactory.java new file mode 100644 index 0000000000..7507d4b343 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlServerDataSourceChannelFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sqlserver; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +public class SqlServerDataSourceChannelFactory implements DataSourceChannelFactory { + @Override + public String getName() { + return "sqlserver"; + } + + @Override + public DataSourceChannel create() { + return new SqlServerDataSourceChannel(); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourceClient.java new file mode 100644 index 0000000000..2a90684f53 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourceClient.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sqlserver; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; + +public class SqlserverDataSourceClient extends CommonDataSourceClient { + + public SqlserverDataSourceClient(BaseConnectionParam baseConnectionParam) { + super(baseConnectionParam); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourcePlugin.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourcePlugin.java new file mode 100644 index 0000000000..7aff452a47 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourcePlugin.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sqlserver; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class SqlserverDataSourcePlugin implements DolphinSchedulerPlugin { + @Override + public Iterable getDatasourceChannelFactorys() { + return ImmutableList.of(new SqlServerDataSourceChannelFactory()); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourceChannelFactoryTest.java new file mode 100644 index 0000000000..bfa4df68e9 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourceChannelFactoryTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sqlserver; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; + +import org.junit.Assert; +import org.junit.Test; + +public class SqlserverDataSourceChannelFactoryTest { + + @Test + public void testCreate() { + SqlServerDataSourceChannelFactory sourceChannelFactory = new SqlServerDataSourceChannelFactory(); + DataSourceChannel dataSourceChannel = sourceChannelFactory.create(); + Assert.assertNotNull(dataSourceChannel); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourceChannelTest.java new file mode 100644 index 0000000000..e9c18112db --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourceChannelTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sqlserver; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SqlServerConnectionParam; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient") +@PrepareForTest({SqlserverDataSourceClient.class, SqlServerDataSourceChannel.class}) +public class SqlserverDataSourceChannelTest { + + @Test + public void testCreateDataSourceClient() { + SqlServerDataSourceChannel sourceChannel = PowerMockito.mock(SqlServerDataSourceChannel.class); + SqlserverDataSourceClient dataSourceClient = PowerMockito.mock(SqlserverDataSourceClient.class); + PowerMockito.when(sourceChannel.createDataSourceClient(Mockito.any())).thenReturn(dataSourceClient); + Assert.assertNotNull(sourceChannel.createDataSourceClient(new SqlServerConnectionParam())); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourcePluginTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourcePluginTest.java new file mode 100644 index 0000000000..89ab6faee5 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SqlserverDataSourcePluginTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sqlserver; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import org.junit.Assert; +import org.junit.Test; + +public class SqlserverDataSourcePluginTest { + + private SqlserverDataSourcePlugin dataSourcePlugin = new SqlserverDataSourcePlugin(); + + @Test + public void testGetDatasourceChannelFactorys() { + Iterable channelFactorys = dataSourcePlugin.getDatasourceChannelFactorys(); + for (DataSourceChannelFactory dataSourceChannelFactory : channelFactorys) { + Assert.assertTrue(dataSourceChannelFactory instanceof DataSourceChannelFactory); + } + } + +} diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml new file mode 100644 index 0000000000..064036facb --- /dev/null +++ b/dolphinscheduler-datasource-plugin/pom.xml @@ -0,0 +1,42 @@ + + + + + dolphinscheduler + org.apache.dolphinscheduler + 2.0.0-SNAPSHOT + + 4.0.0 + dolphinscheduler-datasource-plugin + ${project.artifactId} + pom + + + dolphinscheduler-datasource-sqlserver + dolphinscheduler-datasource-clickhouse + dolphinscheduler-datasource-db2 + dolphinscheduler-datasource-hive + dolphinscheduler-datasource-mysql + dolphinscheduler-datasource-oracle + dolphinscheduler-datasource-postgresql + dolphinscheduler-datasource-api + + + \ No newline at end of file diff --git a/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml b/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml index 85ffa5a270..979d332ecf 100644 --- a/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml +++ b/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml @@ -100,4 +100,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 50d2eabf64..89faf0e477 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.server.worker.runner.RetryReportTaskStatusThr import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.apache.dolphinscheduler.spi.exception.PluginNotFoundException; +import org.apache.dolphinscheduler.spi.exception.PluginException; import org.apache.dolphinscheduler.spi.plugin.DolphinPluginLoader; import org.apache.dolphinscheduler.spi.plugin.DolphinPluginManagerConfig; import org.apache.dolphinscheduler.spi.utils.StringUtils; @@ -192,7 +192,7 @@ public class WorkerServer implements IStoppable { throw new RuntimeException("Load Task Plugin Failed !", e); } if (MapUtils.isEmpty(taskPluginManager.getTaskChannelMap())) { - throw new PluginNotFoundException("Task Plugin Not Found,Please Check Config File"); + throw new PluginException("Task Plugin Not Found,Please Check Config File"); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index a8c823b119..87dcd16639 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -41,7 +41,7 @@ import org.apache.dolphinscheduler.server.worker.plugin.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext; -import org.apache.dolphinscheduler.spi.exception.PluginNotFoundException; +import org.apache.dolphinscheduler.spi.exception.PluginException; import org.apache.dolphinscheduler.spi.task.AbstractTask; import org.apache.dolphinscheduler.spi.task.TaskAlertInfo; import org.apache.dolphinscheduler.spi.task.TaskChannel; @@ -171,7 +171,7 @@ public class TaskExecuteThread implements Runnable, Delayed { TaskChannel taskChannel = taskPluginManager.getTaskChannelMap().get(taskExecutionContext.getTaskType()); if (null == taskChannel) { - throw new PluginNotFoundException(String.format("%s Task Plugin Not Found,Please Check Config File.", taskExecutionContext.getTaskType())); + throw PluginException.getInstance(String.format("%s Task Plugin Not Found,Please Check Config File.", taskExecutionContext.getTaskType())); } TaskRequest taskRequest = JSONUtils.parseObject(JSONUtils.toJsonString(taskExecutionContext), TaskRequest.class); String taskLogName = LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java index dd190f771c..f9d51a910e 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.server.master.consumer; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskType; @@ -34,6 +33,7 @@ import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.queue.TaskPriority; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; +import org.apache.dolphinscheduler.spi.enums.DbType; import java.util.Date; import java.util.concurrent.TimeUnit; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java index b4e41cb06a..e7afa143bb 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.server.master.runner.task; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.spi.enums.ResourceType; @@ -33,6 +32,7 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.task.request.DataxTaskExecutionContext; import java.util.ArrayList; diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/DolphinSchedulerPlugin.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/DolphinSchedulerPlugin.java index fe327b6400..272e0d4699 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/DolphinSchedulerPlugin.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/DolphinSchedulerPlugin.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.spi; import static java.util.Collections.emptyList; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; /** @@ -39,4 +40,12 @@ public interface DolphinSchedulerPlugin { return emptyList(); } + /** + * get datasource plugin factory + * @return datasource factory + */ + default Iterable getDatasourceChannelFactorys() { + return emptyList(); + } + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/BaseConnectionParam.java similarity index 52% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseConnectionParam.java rename to dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/BaseConnectionParam.java index 44a5962f92..7fa7a7bcb0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseConnectionParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/BaseConnectionParam.java @@ -15,17 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.dolphinscheduler.plugin.task.datasource.clickhouse.ClickhouseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.db2.Db2ConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.hive.HiveConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.mysql.MysqlConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.oracle.OracleConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.postgresql.PostgreSqlConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.presto.PrestoConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.spark.SparkConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.sqlserver.SqlServerConnectionParam; +package org.apache.dolphinscheduler.spi.datasource; + +import java.util.HashMap; +import java.util.Map; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; @@ -33,15 +26,15 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; /** * The base model of connection param * - * {@link ClickhouseConnectionParam} - * {@link Db2ConnectionParam} - * {@link HiveConnectionParam} - * {@link MysqlConnectionParam} - * {@link OracleConnectionParam} - * {@link PostgreSqlConnectionParam} - * {@link PrestoConnectionParam} - * {@link SparkConnectionParam} - * {@link SqlServerConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickhouseConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2ConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSqlConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SqlServerConnectionParam} */ @JsonInclude(Include.NON_NULL) public abstract class BaseConnectionParam implements ConnectionParam { @@ -56,8 +49,16 @@ public abstract class BaseConnectionParam implements ConnectionParam { protected String jdbcUrl; + protected String driverLocation; + + protected String driverClassName; + + protected String validationQuery; + protected String other; + private Map props = new HashMap<>(); + public String getUser() { return user; } @@ -98,6 +99,30 @@ public abstract class BaseConnectionParam implements ConnectionParam { this.jdbcUrl = jdbcUrl; } + public String getDriverLocation() { + return driverLocation; + } + + public void setDriverLocation(String driverLocation) { + this.driverLocation = driverLocation; + } + + public String getDriverClassName() { + return driverClassName; + } + + public void setDriverClassName(String driverClassName) { + this.driverClassName = driverClassName; + } + + public String getValidationQuery() { + return validationQuery; + } + + public void setValidationQuery(String validationQuery) { + this.validationQuery = validationQuery; + } + public String getOther() { return other; } @@ -105,4 +130,12 @@ public abstract class BaseConnectionParam implements ConnectionParam { public void setOther(String other) { this.other = other; } + + public Map getProps() { + return props; + } + + public void setProps(Map props) { + this.props = props; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/ConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/ConnectionParam.java similarity index 94% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/ConnectionParam.java rename to dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/ConnectionParam.java index d4ec697751..b3eb903dc7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/ConnectionParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/ConnectionParam.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource; +package org.apache.dolphinscheduler.spi.datasource; import java.io.Serializable; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/ConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannel.java similarity index 80% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/ConnectionParam.java rename to dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannel.java index 46bd979a2d..5328f9599a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/ConnectionParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannel.java @@ -15,12 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource; +package org.apache.dolphinscheduler.spi.datasource; -import java.io.Serializable; +public interface DataSourceChannel { -/** - * The model of Datasource Connection param - */ -public interface ConnectionParam extends Serializable { + DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam); } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java new file mode 100644 index 0000000000..c947c3a647 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.datasource; + +public interface DataSourceChannelFactory { + /** + * get datasource client + */ + DataSourceChannel create(); + + /** + * get registry component name + */ + String getName(); +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceClient.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceClient.java new file mode 100644 index 0000000000..879d198284 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceClient.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.datasource; + +import java.sql.Connection; + +public interface DataSourceClient { + + void checkClient(); + + void close(); + + Connection getConnection(); +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 0c88407a02..3809c52269 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -22,31 +22,38 @@ import static java.util.stream.Collectors.toMap; import java.util.Arrays; import java.util.Map; +import com.baomidou.mybatisplus.annotation.EnumValue; import com.google.common.base.Functions; public enum DbType { + MYSQL(0, "mysql"), + POSTGRESQL(1, "postgresql"), + HIVE(2, "hive"), + SPARK(3, "spark"), + CLICKHOUSE(4, "clickhouse"), + ORACLE(5, "oracle"), + SQLSERVER(6, "sqlserver"), + DB2(7, "db2"), + PRESTO(8, "presto"), + H2(9, "h2"); - MYSQL(0), - POSTGRESQL(1), - HIVE(2), - SPARK(3), - CLICKHOUSE(4), - ORACLE(5), - SQLSERVER(6), - DB2(7), - PRESTO(8), - H2(9); - - DbType(int code) { + @EnumValue + private final int code; + private final String descp; + + DbType(int code, String descp) { this.code = code; + this.descp = descp; } - private final int code; - public int getCode() { return code; } + public String getDescp() { + return descp; + } + private static final Map DB_TYPE_MAP = Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); @@ -56,4 +63,5 @@ public enum DbType { } return null; } + } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginNotFoundException.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginException.java similarity index 69% rename from dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginNotFoundException.java rename to dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginException.java index 2153299f17..4e7dcf3db7 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginNotFoundException.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginException.java @@ -17,15 +17,23 @@ package org.apache.dolphinscheduler.spi.exception; -public class PluginNotFoundException extends RuntimeException { +public class PluginException extends RuntimeException { private static final long serialVersionUID = -5487812425126112159L; - public PluginNotFoundException(String message, Throwable cause) { + public PluginException(String message, Throwable cause) { super(message, cause); } - public PluginNotFoundException(String message) { + public PluginException(String message) { super(message); } + + public static PluginException getInstance(String message) { + return new PluginException(message); + } + + public static PluginException getInstance(String message, Throwable cause) { + return new PluginException(message + ": " + cause.getMessage(), cause); + } } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java index a3a8b77564..b2eb17bd54 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java @@ -72,4 +72,140 @@ public class Constants { */ public static final String YYYYMMDDHHMMSSSSS = "yyyyMMddHHmmssSSS"; + public static final String SPRING_DATASOURCE_MIN_IDLE = "spring.datasource.minIdle"; + + public static final String SPRING_DATASOURCE_MAX_ACTIVE = "spring.datasource.maxActive"; + + public static final String SPRING_DATASOURCE_TEST_ON_BORROW = "spring.datasource.testOnBorrow"; + + /** + * datasource plugin conf + */ + + public static final String DATASOURCE_PLUGIN_BINDING = "datasource.plugin.binding"; + + public static final String DATASOURCE_PLUGIN_DIR = "datasource.plugin.dir"; + + public static final String MAVEN_LOCAL_REPOSITORY = "maven.local.repository"; + + /** + * default datasource plugin dir + */ + public static final String DATASOURCE_PLUGIN_PATH = "lib/plugin/datasource"; + + /** + * java.security.krb5.conf + */ + public static final String JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + + /** + * java.security.krb5.conf.path + */ + public static final String JAVA_SECURITY_KRB5_CONF_PATH = "java.security.krb5.conf.path"; + + /** + * hadoop.security.authentication + */ + public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; + + /** + * hadoop.security.authentication + */ + public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state"; + + /** + * loginUserFromKeytab user + */ + public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username"; + + /** + * loginUserFromKeytab path + */ + public static final String LOGIN_USER_KEY_TAB_PATH = "login.user.keytab.path"; + + /** + * resource storage type + */ + public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type"; + + /** + * kerberos + */ + public static final String KERBEROS = "kerberos"; + + /** + * driver + */ + public static final String ORG_POSTGRESQL_DRIVER = "org.postgresql.Driver"; + public static final String COM_MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver"; + public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"; + public static final String COM_CLICKHOUSE_JDBC_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; + public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.OracleDriver"; + public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; + public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"; + + + /** + * validation Query + */ + public static final String POSTGRESQL_VALIDATION_QUERY = "select version()"; + public static final String MYSQL_VALIDATION_QUERY = "select 1"; + public static final String HIVE_VALIDATION_QUERY = "select 1"; + public static final String CLICKHOUSE_VALIDATION_QUERY = "select 1"; + public static final String ORACLE_VALIDATION_QUERY = "select 1 from dual"; + public static final String SQLSERVER_VALIDATION_QUERY = "select 1"; + public static final String DB2_VALIDATION_QUERY = "select 1 from sysibm.sysdummy1"; + public static final String PRESTO_VALIDATION_QUERY = "select 1"; + + /** + * jdbc url + */ + public static final String JDBC_MYSQL = "jdbc:mysql://"; + public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; + public static final String JDBC_HIVE_2 = "jdbc:hive2://"; + public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://"; + public static final String JDBC_ORACLE_SID = "jdbc:oracle:thin:@"; + public static final String JDBC_ORACLE_SERVICE_NAME = "jdbc:oracle:thin:@//"; + public static final String JDBC_SQLSERVER = "jdbc:sqlserver://"; + public static final String JDBC_DB2 = "jdbc:db2://"; + public static final String JDBC_PRESTO = "jdbc:presto://"; + + + public static final String ADDRESS = "address"; + public static final String DATABASE = "database"; + public static final String JDBC_URL = "jdbcUrl"; + public static final String PRINCIPAL = "principal"; + public static final String OTHER = "other"; + public static final String ORACLE_DB_CONNECT_TYPE = "connectType"; + public static final String KERBEROS_KRB5_CONF_PATH = "javaSecurityKrb5Conf"; + public static final String KERBEROS_KEY_TAB_USERNAME = "loginUserKeytabUsername"; + public static final String KERBEROS_KEY_TAB_PATH = "loginUserKeytabPath"; + + /** + * DOUBLE_SLASH // + */ + public static final String DOUBLE_SLASH = "//"; + + /** + * comma , + */ + public static final String COMMA = ","; + + /** + * COLON : + */ + public static final String COLON = ":"; + + /** + * AT SIGN + */ + public static final String AT_SIGN = "@"; + + /** + * datasource encryption salt + */ + public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*"; + public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable"; + public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt"; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/AbstractDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/AbstractDatasourceProcessor.java deleted file mode 100644 index 19d88e1588..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/AbstractDatasourceProcessor.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.commons.collections.MapUtils; - -import java.util.Map; -import java.util.regex.Pattern; - -public abstract class AbstractDatasourceProcessor implements DatasourceProcessor { - - private static final Pattern IPV4_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); - - private static final Pattern IPV6_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.\\:\\[\\]]+$"); - - private static final Pattern DATABASE_PATTER = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); - - private static final Pattern PARAMS_PATTER = Pattern.compile("^[a-zA-Z0-9\\-\\_\\/]+$"); - - @Override - public void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { - checkHost(baseDataSourceParamDTO.getHost()); - checkDatasourcePatter(baseDataSourceParamDTO.getDatabase()); - checkOther(baseDataSourceParamDTO.getOther()); - } - - /** - * Check the host is valid - * - * @param host datasource host - */ - protected void checkHost(String host) { - if (!IPV4_PATTERN.matcher(host).matches() || !IPV6_PATTERN.matcher(host).matches()) { - throw new IllegalArgumentException("datasource host illegal"); - } - } - - /** - * check database name is valid - * - * @param database database name - */ - protected void checkDatasourcePatter(String database) { - if (!DATABASE_PATTER.matcher(database).matches()) { - throw new IllegalArgumentException("datasource name illegal"); - } - } - - /** - * check other is valid - * - * @param other other - */ - protected void checkOther(Map other) { - if (MapUtils.isEmpty(other)) { - return; - } - boolean paramsCheck = other.entrySet().stream().allMatch(p -> PARAMS_PATTER.matcher(p.getValue()).matches()); - if (!paramsCheck) { - throw new IllegalArgumentException("datasource other params illegal"); - } - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseDataSourceParamDTO.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseDataSourceParamDTO.java deleted file mode 100644 index 5af330d38e..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseDataSourceParamDTO.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.dolphinscheduler.plugin.task.datasource.clickhouse.ClickHouseDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.db2.Db2DatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.hive.HiveDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.mysql.MysqlDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.oracle.OracleDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.postgresql.PostgreSqlDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.presto.PrestoDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.spark.SparkDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.sqlserver.SqlServerDatasourceParamDTO; -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.io.Serializable; -import java.util.Map; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -/** - * Basic datasource params submitted to api. - * - * see {@link MysqlDatasourceParamDTO} - * see {@link PostgreSqlDatasourceParamDTO} - * see {@link HiveDataSourceParamDTO} - * see {@link SparkDatasourceParamDTO} - * see {@link ClickHouseDatasourceParamDTO} - * see {@link OracleDatasourceParamDTO} - * see {@link SqlServerDatasourceParamDTO} - * see {@link Db2DatasourceParamDTO} - * see {@link PrestoDatasourceParamDTO} - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -@JsonSubTypes(value = { - @JsonSubTypes.Type(value = MysqlDatasourceParamDTO.class, name = "MYSQL"), - @JsonSubTypes.Type(value = PostgreSqlDatasourceParamDTO.class, name = "POSTGRESQL"), - @JsonSubTypes.Type(value = HiveDataSourceParamDTO.class, name = "HIVE"), - @JsonSubTypes.Type(value = SparkDatasourceParamDTO.class, name = "SPARK"), - @JsonSubTypes.Type(value = ClickHouseDatasourceParamDTO.class, name = "CLICKHOUSE"), - @JsonSubTypes.Type(value = OracleDatasourceParamDTO.class, name = "ORACLE"), - @JsonSubTypes.Type(value = SqlServerDatasourceParamDTO.class, name = "SQLSERVER"), - @JsonSubTypes.Type(value = Db2DatasourceParamDTO.class, name = "DB2"), - @JsonSubTypes.Type(value = PrestoDatasourceParamDTO.class, name = "PRESTO"), -}) -public abstract class BaseDataSourceParamDTO implements Serializable { - - protected Integer id; - - protected String name; - - protected String note; - - protected String host; - - protected Integer port; - - protected String database; - - protected String userName; - - protected String password; - - protected Map other; - - public Integer getId() { - return id; - } - - public void setId(Integer id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getNote() { - return note; - } - - public void setNote(String note) { - this.note = note; - } - - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host; - } - - public Integer getPort() { - return port; - } - - public void setPort(Integer port) { - this.port = port; - } - - public String getDatabase() { - return database; - } - - public void setDatabase(String database) { - this.database = database; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public Map getOther() { - return other; - } - - public void setOther(Map other) { - this.other = other; - } - - /** - * Get the datasource type - * see{@link DbType} - * - * @return datasource type code - */ - public abstract DbType getType(); -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceUtil.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceUtil.java deleted file mode 100644 index d7b47f07ac..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceUtil.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.dolphinscheduler.plugin.task.datasource.clickhouse.ClickHouseDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.db2.Db2DatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.hive.HiveDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.mysql.MysqlDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.oracle.OracleDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.postgresql.PostgreSqlDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.presto.PrestoDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.spark.SparkDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.sqlserver.SqlServerDatasourceProcessor; -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.sql.Connection; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DatasourceUtil { - - private DatasourceUtil() { - } - - private static final Logger logger = LoggerFactory.getLogger(DatasourceUtil.class); - - private static final DatasourceProcessor mysqlProcessor = new MysqlDatasourceProcessor(); - private static final DatasourceProcessor postgreSqlProcessor = new PostgreSqlDatasourceProcessor(); - private static final DatasourceProcessor hiveProcessor = new HiveDatasourceProcessor(); - private static final DatasourceProcessor sparkProcessor = new SparkDatasourceProcessor(); - private static final DatasourceProcessor clickhouseProcessor = new ClickHouseDatasourceProcessor(); - private static final DatasourceProcessor oracleProcessor = new OracleDatasourceProcessor(); - private static final DatasourceProcessor sqlServerProcessor = new SqlServerDatasourceProcessor(); - private static final DatasourceProcessor db2PROCESSOR = new Db2DatasourceProcessor(); - private static final DatasourceProcessor prestoPROCESSOR = new PrestoDatasourceProcessor(); - - public static ConnectionParam buildConnectionParams(DbType dbType, String connectionJson) { - return getDatasourceProcessor(dbType).createConnectionParams(connectionJson); - } - - public static Connection getConnection(DbType dbType, ConnectionParam connectionParam) { - try { - return getDatasourceProcessor(dbType).getConnection(connectionParam); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static String getJdbcUrl(DbType dbType, ConnectionParam baseConnectionParam) { - return getDatasourceProcessor(dbType).getJdbcUrl(baseConnectionParam); - } - - public static DatasourceProcessor getDatasourceProcessor(DbType dbType) { - switch (dbType) { - case MYSQL: - return mysqlProcessor; - case POSTGRESQL: - return postgreSqlProcessor; - case HIVE: - return hiveProcessor; - case SPARK: - return sparkProcessor; - case CLICKHOUSE: - return clickhouseProcessor; - case ORACLE: - return oracleProcessor; - case SQLSERVER: - return sqlServerProcessor; - case DB2: - return db2PROCESSOR; - case PRESTO: - return prestoPROCESSOR; - default: - throw new IllegalArgumentException("datasource type illegal:" + dbType); - } - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/HiveConfUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/HiveConfUtils.java deleted file mode 100644 index 0985b7457c..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/HiveConfUtils.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * hive conf utils - */ -public class HiveConfUtils { - - private HiveConfUtils() { - throw new UnsupportedOperationException("Construct HiveConfUtils"); - } - - private static class HiveConfHandler { - private static HiveConf singleton; - - private static Map hiveConfVars; - - static { - singleton = new HiveConf(); - hiveConfVars = new HashMap<>(); - Arrays.stream(ConfVars.values()).forEach(confVar -> hiveConfVars.put(confVar.varname,confVar)); - } - } - - /** - * get HiveConf instance - * @return HiveConf hiveConf - */ - public static HiveConf getInstance() { - return HiveConfHandler.singleton; - } - - /** - * get hive conf vars - * @return - */ - public static Map getHiveConfVars() { - return HiveConfHandler.hiveConfVars; - } - - /** - * Determine if it belongs to a hive conf property - * @param conf config - * @return boolean result - */ - public static boolean isHiveConfVar(String conf) { - // the default hive conf var name - String confKey = conf.split("=")[0]; - Map hiveConfVars = HiveConfUtils.getHiveConfVars(); - if (hiveConfVars.get(confKey) != null) { - return true; - } - - // the security authorization hive conf var name - HiveConf hiveConf = HiveConfUtils.getInstance(); - String hiveAuthorizationSqlStdAuthConfigWhitelist = hiveConf.getVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST); - Pattern modWhiteListPattern = Pattern.compile(hiveAuthorizationSqlStdAuthConfigWhitelist); - Matcher matcher = modWhiteListPattern.matcher(confKey); - return matcher.matches(); - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java deleted file mode 100644 index 624afee940..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.clickhouse; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_CLICKHOUSE_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_CLICKHOUSE; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) createConnectionParams(connectionJson); - - ClickHouseDatasourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDatasourceParamDTO(); - clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser()); - clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - clickHouseDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - clickHouseDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - - return clickHouseDatasourceParamDTO; - } - - @Override - public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - ClickHouseDatasourceParamDTO clickHouseParam = (ClickHouseDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_CLICKHOUSE, clickHouseParam.getHost(), clickHouseParam.getPort()); - String jdbcUrl = address + "/" + clickHouseParam.getDatabase(); - - ClickhouseConnectionParam clickhouseConnectionParam = new ClickhouseConnectionParam(); - clickhouseConnectionParam.setDatabase(clickHouseParam.getDatabase()); - clickhouseConnectionParam.setAddress(address); - clickhouseConnectionParam.setJdbcUrl(jdbcUrl); - clickhouseConnectionParam.setUser(clickHouseParam.getUserName()); - clickhouseConnectionParam.setPassword(encodePassword(clickHouseParam.getPassword())); - clickhouseConnectionParam.setOther(transformOther(clickHouseParam.getOther())); - return clickhouseConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, ClickhouseConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_CLICKHOUSE_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; - String jdbcUrl = clickhouseConnectionParam.getJdbcUrl(); - if (StringUtils.isNotEmpty(clickhouseConnectionParam.getOther())) { - jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther()); - } - return jdbcUrl; - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam), - clickhouseConnectionParam.getUser(), decodePassword(clickhouseConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.CLICKHOUSE; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, "&"))); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (other == null) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split("&"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceProcessor.java deleted file mode 100644 index ee130ebe60..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceProcessor.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.db2; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_DB2_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_DB2; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class Db2DatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - Db2ConnectionParam connectionParams = (Db2ConnectionParam) createConnectionParams(connectionJson); - - Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO(); - db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName()); - - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - db2DatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - db2DatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return db2DatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - Db2DatasourceParamDTO db2Param = (Db2DatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_DB2, db2Param.getHost(), db2Param.getPort()); - String jdbcUrl = String.format("%s/%s", address, db2Param.getDatabase()); - - Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam(); - db2ConnectionParam.setAddress(address); - db2ConnectionParam.setDatabase(db2Param.getDatabase()); - db2ConnectionParam.setJdbcUrl(jdbcUrl); - db2ConnectionParam.setUser(db2Param.getUserName()); - db2ConnectionParam.setPassword(encodePassword(db2Param.getPassword())); - db2ConnectionParam.setOther(transformOther(db2Param.getOther())); - - return db2ConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, Db2ConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_DB2_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(db2ConnectionParam.getOther())) { - return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther()); - } - return db2ConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(db2ConnectionParam), - db2ConnectionParam.getUser(), decodePassword(db2ConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.DB2; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, ";"))); - stringBuilder.deleteCharAt(stringBuilder.length() - 1); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (other == null) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split("&")) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDatasourceProcessor.java deleted file mode 100644 index 16c4ceb111..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDatasourceProcessor.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.hive; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_HIVE_2; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.HiveConfUtils; -import org.apache.dolphinscheduler.plugin.task.util.CommonUtils; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); - HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) createConnectionParams(connectionJson); - - hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase()); - hiveDataSourceParamDTO.setUserName(hiveConnectionParam.getUser()); - hiveDataSourceParamDTO.setOther(parseOther(hiveConnectionParam.getOther())); - hiveDataSourceParamDTO.setLoginUserKeytabUsername(hiveConnectionParam.getLoginUserKeytabUsername()); - hiveDataSourceParamDTO.setLoginUserKeytabPath(hiveConnectionParam.getLoginUserKeytabPath()); - hiveDataSourceParamDTO.setJavaSecurityKrb5Conf(hiveConnectionParam.getJavaSecurityKrb5Conf()); - - String[] tmpArray = hiveConnectionParam.getAddress().split(DOUBLE_SLASH); - StringBuilder hosts = new StringBuilder(); - String[] hostPortArray = tmpArray[tmpArray.length - 1].split(COMMA); - for (String hostPort : hostPortArray) { - hosts.append(hostPort.split(COLON)[0]).append(COMMA); - } - hosts.deleteCharAt(hosts.length() - 1); - hiveDataSourceParamDTO.setHost(hosts.toString()); - hiveDataSourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return hiveDataSourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - HiveDataSourceParamDTO hiveParam = (HiveDataSourceParamDTO) datasourceParam; - StringBuilder address = new StringBuilder(); - address.append(JDBC_HIVE_2); - for (String zkHost : hiveParam.getHost().split(",")) { - address.append(String.format("%s:%s,", zkHost, hiveParam.getPort())); - } - address.deleteCharAt(address.length() - 1); - String jdbcUrl = address.toString() + "/" + hiveParam.getDatabase(); - if (CommonUtils.getKerberosStartupState()) { - jdbcUrl += ";principal=" + hiveParam.getPrincipal(); - } - - HiveConnectionParam hiveConnectionParam = new HiveConnectionParam(); - hiveConnectionParam.setDatabase(hiveParam.getDatabase()); - hiveConnectionParam.setAddress(address.toString()); - hiveConnectionParam.setJdbcUrl(jdbcUrl); - hiveConnectionParam.setUser(hiveParam.getUserName()); - hiveConnectionParam.setPassword(encodePassword(hiveParam.getPassword())); - - if (CommonUtils.getKerberosStartupState()) { - hiveConnectionParam.setPrincipal(hiveParam.getPrincipal()); - hiveConnectionParam.setJavaSecurityKrb5Conf(hiveParam.getJavaSecurityKrb5Conf()); - hiveConnectionParam.setLoginUserKeytabPath(hiveParam.getLoginUserKeytabPath()); - hiveConnectionParam.setLoginUserKeytabUsername(hiveParam.getLoginUserKeytabUsername()); - } - hiveConnectionParam.setOther(transformOther(hiveParam.getOther())); - return hiveConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, HiveConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; - String jdbcUrl = hiveConnectionParam.getJdbcUrl(); - String otherParams = filterOther(hiveConnectionParam.getOther()); - if (StringUtils.isNotEmpty(otherParams) && !"?".equals(otherParams.substring(0, 1))) { - jdbcUrl += ";"; - } - return jdbcUrl + otherParams; - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { - HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; - CommonUtils.loadKerberosConf(hiveConnectionParam.getJavaSecurityKrb5Conf(), - hiveConnectionParam.getLoginUserKeytabUsername(), hiveConnectionParam.getLoginUserKeytabPath()); - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), - hiveConnectionParam.getUser(), decodePassword(hiveConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.HIVE; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); - return stringBuilder.toString(); - } - - private String filterOther(String otherParams) { - if (StringUtils.isBlank(otherParams)) { - return ""; - } - - StringBuilder hiveConfListSb = new StringBuilder(); - hiveConfListSb.append("?"); - StringBuilder sessionVarListSb = new StringBuilder(); - - String[] otherArray = otherParams.split(";", -1); - - for (String conf : otherArray) { - if (HiveConfUtils.isHiveConfVar(conf)) { - hiveConfListSb.append(conf).append(";"); - } else { - sessionVarListSb.append(conf).append(";"); - } - } - - // remove the last ";" - if (sessionVarListSb.length() > 0) { - sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); - } - - if (hiveConfListSb.length() > 0) { - hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); - } - - return sessionVarListSb.toString() + hiveConfListSb.toString(); - } - - private Map parseOther(String other) { - if (other == null) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split(";"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceProcessor.java deleted file mode 100644 index e41ccd21c5..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceProcessor.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.mysql; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_MYSQL_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_MYSQL; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor { - - private final Logger logger = LoggerFactory.getLogger(MysqlDatasourceProcessor.class); - - private static final String ALLOW_LOAD_LOCAL_IN_FILE_NAME = "allowLoadLocalInfile"; - - private static final String AUTO_DESERIALIZE = "autoDeserialize"; - - private static final String ALLOW_LOCAL_IN_FILE_NAME = "allowLocalInfile"; - - private static final String ALLOW_URL_IN_LOCAL_IN_FILE_NAME = "allowUrlInLocalInfile"; - - private static final String APPEND_PARAMS = "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - MysqlConnectionParam connectionParams = (MysqlConnectionParam) createConnectionParams(connectionJson); - MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); - - mysqlDatasourceParamDTO.setUserName(connectionParams.getUser()); - mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - mysqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String address = connectionParams.getAddress(); - String[] hostSeperator = address.split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - mysqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - mysqlDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - - return mysqlDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { - MysqlDatasourceParamDTO mysqlDatasourceParam = (MysqlDatasourceParamDTO) dataSourceParam; - String address = String.format("%s%s:%s", JDBC_MYSQL, mysqlDatasourceParam.getHost(), mysqlDatasourceParam.getPort()); - String jdbcUrl = String.format("%s/%s", address, mysqlDatasourceParam.getDatabase()); - - MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); - mysqlConnectionParam.setJdbcUrl(jdbcUrl); - mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase()); - mysqlConnectionParam.setAddress(address); - mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName()); - mysqlConnectionParam.setPassword(encodePassword(mysqlDatasourceParam.getPassword())); - mysqlConnectionParam.setOther(transformOther(mysqlDatasourceParam.getOther())); - - return mysqlConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, MysqlConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_MYSQL_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; - String jdbcUrl = mysqlConnectionParam.getJdbcUrl(); - if (StringUtils.isNotEmpty(mysqlConnectionParam.getOther())) { - return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS); - } - return String.format("%s?%s", jdbcUrl, APPEND_PARAMS); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - String user = mysqlConnectionParam.getUser(); - if (user.contains(AUTO_DESERIALIZE)) { - logger.warn("sensitive param : {} in username field is filtered", AUTO_DESERIALIZE); - user = user.replace(AUTO_DESERIALIZE, ""); - } - String password = decodePassword(mysqlConnectionParam.getPassword()); - if (password.contains(AUTO_DESERIALIZE)) { - logger.warn("sensitive param : {} in password field is filtered", AUTO_DESERIALIZE); - password = password.replace(AUTO_DESERIALIZE, ""); - } - return DriverManager.getConnection(getJdbcUrl(connectionParam), user, password); - } - - @Override - public DbType getDbType() { - return DbType.MYSQL; - } - - private String transformOther(Map paramMap) { - if (MapUtils.isEmpty(paramMap)) { - return null; - } - Map otherMap = new HashMap<>(); - paramMap.forEach((k, v) -> { - if (!checkKeyIsLegitimate(k)) { - return; - } - otherMap.put(k, v); - }); - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); - return stringBuilder.toString(); - } - - private static boolean checkKeyIsLegitimate(String key) { - return !key.contains(ALLOW_LOAD_LOCAL_IN_FILE_NAME) - && !key.contains(AUTO_DESERIALIZE) - && !key.contains(ALLOW_LOCAL_IN_FILE_NAME) - && !key.contains(ALLOW_URL_IN_LOCAL_IN_FILE_NAME); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split("&")) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceProcessor.java deleted file mode 100644 index bfe94960b0..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceProcessor.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.oracle; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.AT_SIGN; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_ORACLE_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_ORACLE_SERVICE_NAME; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_ORACLE_SID; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbConnectType; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public class OracleDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - OracleConnectionParam connectionParams = (OracleConnectionParam) createConnectionParams(connectionJson); - OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO(); - - oracleDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - oracleDatasourceParamDTO.setUserName(connectionParams.getUser()); - oracleDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String hostSeperator = DOUBLE_SLASH; - if (DbConnectType.ORACLE_SID.equals(connectionParams.connectType)) { - hostSeperator = AT_SIGN; - } - String[] hostPort = connectionParams.getAddress().split(hostSeperator); - String[] hostPortArray = hostPort[hostPort.length - 1].split(COMMA); - oracleDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - oracleDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - - return oracleDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - OracleDatasourceParamDTO oracleParam = (OracleDatasourceParamDTO) datasourceParam; - String address; - if (DbConnectType.ORACLE_SID.equals(oracleParam.getConnectType())) { - address = String.format("%s%s:%s", - JDBC_ORACLE_SID, oracleParam.getHost(), oracleParam.getPort()); - } else { - address = String.format("%s%s:%s", - JDBC_ORACLE_SERVICE_NAME, oracleParam.getHost(), oracleParam.getPort()); - } - String jdbcUrl = address + "/" + oracleParam.getDatabase(); - - OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); - oracleConnectionParam.setUser(oracleParam.getUserName()); - oracleConnectionParam.setPassword(encodePassword(oracleParam.getPassword())); - oracleConnectionParam.setAddress(address); - oracleConnectionParam.setJdbcUrl(jdbcUrl); - oracleConnectionParam.setDatabase(oracleParam.getDatabase()); - oracleConnectionParam.setConnectType(oracleParam.getConnectType()); - oracleConnectionParam.setOther(transformOther(oracleParam.getOther())); - - return oracleConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, OracleConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_ORACLE_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(oracleConnectionParam.getOther())) { - return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther()); - } - return oracleConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), - oracleConnectionParam.getUser(), decodePassword(oracleConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.ORACLE; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - List list = new ArrayList<>(); - otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); - return String.join("&", list); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split("&"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlConnectionParam.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlConnectionParam.java deleted file mode 100644 index 1d459ca2b0..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlConnectionParam.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.postgresql; - -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; - -public class PostgreSqlConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "PostgreSqlConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java deleted file mode 100644 index fc705c6e41..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.postgresql; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_POSTGRESQL; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_POSTGRESQL_DRIVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) createConnectionParams(connectionJson); - PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO(); - postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser()); - postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String address = connectionParams.getAddress(); - String[] hostSeperator = address.split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - postgreSqlDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - postgreSqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return postgreSqlDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - PostgreSqlDatasourceParamDTO postgreSqlParam = (PostgreSqlDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_POSTGRESQL, postgreSqlParam.getHost(), postgreSqlParam.getPort()); - String jdbcUrl = String.format("%s/%s", address, postgreSqlParam.getDatabase()); - - PostgreSqlConnectionParam postgreSqlConnectionParam = new PostgreSqlConnectionParam(); - postgreSqlConnectionParam.setJdbcUrl(jdbcUrl); - postgreSqlConnectionParam.setAddress(address); - postgreSqlConnectionParam.setDatabase(postgreSqlParam.getDatabase()); - postgreSqlConnectionParam.setUser(postgreSqlParam.getUserName()); - postgreSqlConnectionParam.setPassword(encodePassword(postgreSqlParam.getPassword())); - postgreSqlConnectionParam.setOther(transformOther(postgreSqlParam.getOther())); - - return postgreSqlConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, PostgreSqlConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return ORG_POSTGRESQL_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(postgreSqlConnectionParam.getOther())) { - return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther()); - } - return postgreSqlConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam), - postgreSqlConnectionParam.getUser(), decodePassword(postgreSqlConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.POSTGRESQL; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split("&")) { - String[] split = config.split("="); - otherMap.put(split[0], split[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoConnectionParam.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoConnectionParam.java deleted file mode 100644 index b6a226d982..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoConnectionParam.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.presto; - -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; - -public class PrestoConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "PrestoConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceProcessor.java deleted file mode 100644 index d8630bf95f..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceProcessor.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.presto; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_PRESTO_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_PRESTO; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - PrestoConnectionParam connectionParams = (PrestoConnectionParam) createConnectionParams(connectionJson); - - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - - PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO(); - prestoDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - prestoDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - prestoDatasourceParamDTO.setUserName(connectionParams.getUser()); - prestoDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - return prestoDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - PrestoDatasourceParamDTO prestoParam = (PrestoDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_PRESTO, prestoParam.getHost(), prestoParam.getPort()); - String jdbcUrl = address + "/" + prestoParam.getDatabase(); - - PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam(); - prestoConnectionParam.setUser(prestoParam.getUserName()); - prestoConnectionParam.setPassword(encodePassword(prestoParam.getPassword())); - prestoConnectionParam.setOther(transformOther(prestoParam.getOther())); - prestoConnectionParam.setAddress(address); - prestoConnectionParam.setJdbcUrl(jdbcUrl); - prestoConnectionParam.setDatabase(prestoParam.getDatabase()); - - return prestoConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, PrestoConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_PRESTO_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(prestoConnectionParam.getOther())) { - return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther()); - } - return prestoConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), - prestoConnectionParam.getUser(), decodePassword(prestoConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.PRESTO; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isNotEmpty(otherMap)) { - List list = new ArrayList<>(); - otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); - return String.join("&", list); - } - return null; - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split("&"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceProcessor.java deleted file mode 100644 index 00e7ecbe0d..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceProcessor.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.spark; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_HIVE_2; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.plugin.task.util.CommonUtils; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class SparkDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - SparkConnectionParam connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson); - - SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO(); - sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - sparkDatasourceParamDTO.setUserName(connectionParams.getUser()); - sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - sparkDatasourceParamDTO.setJavaSecurityKrb5Conf(connectionParams.getJavaSecurityKrb5Conf()); - sparkDatasourceParamDTO.setLoginUserKeytabPath(connectionParams.getLoginUserKeytabPath()); - sparkDatasourceParamDTO.setLoginUserKeytabUsername(connectionParams.getLoginUserKeytabUsername()); - - StringBuilder hosts = new StringBuilder(); - String[] tmpArray = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = tmpArray[tmpArray.length - 1].split(COMMA); - Arrays.stream(hostPortArray).forEach(hostPort -> hosts.append(hostPort.split(COLON)[0]).append(COMMA)); - hosts.deleteCharAt(hosts.length() - 1); - - sparkDatasourceParamDTO.setHost(hosts.toString()); - sparkDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return sparkDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { - StringBuilder address = new StringBuilder(); - SparkDatasourceParamDTO sparkDatasourceParam = (SparkDatasourceParamDTO) dataSourceParam; - address.append(JDBC_HIVE_2); - for (String zkHost : sparkDatasourceParam.getHost().split(",")) { - address.append(String.format("%s:%s,", zkHost, sparkDatasourceParam.getPort())); - } - address.deleteCharAt(address.length() - 1); - - String jdbcUrl = address + "/" + sparkDatasourceParam.getDatabase(); - if (CommonUtils.getKerberosStartupState()) { - jdbcUrl += ";principal=" + sparkDatasourceParam.getPrincipal(); - } - - SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); - sparkConnectionParam.setPassword(encodePassword(sparkDatasourceParam.getPassword())); - sparkConnectionParam.setUser(sparkDatasourceParam.getUserName()); - sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther())); - sparkConnectionParam.setDatabase(sparkDatasourceParam.getDatabase()); - sparkConnectionParam.setAddress(address.toString()); - sparkConnectionParam.setJdbcUrl(jdbcUrl); - if (CommonUtils.getKerberosStartupState()) { - sparkConnectionParam.setPrincipal(sparkDatasourceParam.getPrincipal()); - sparkConnectionParam.setJavaSecurityKrb5Conf(sparkDatasourceParam.getJavaSecurityKrb5Conf()); - sparkConnectionParam.setLoginUserKeytabPath(sparkDatasourceParam.getLoginUserKeytabPath()); - sparkConnectionParam.setLoginUserKeytabUsername(sparkDatasourceParam.getLoginUserKeytabUsername()); - } - - return sparkConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, SparkConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(sparkConnectionParam.getOther())) { - return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther()); - } - return sparkConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { - SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; - CommonUtils.loadKerberosConf(sparkConnectionParam.getJavaSecurityKrb5Conf(), - sparkConnectionParam.getLoginUserKeytabUsername(), sparkConnectionParam.getLoginUserKeytabPath()); - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(sparkConnectionParam), - sparkConnectionParam.getUser(), decodePassword(sparkConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.SPARK; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - List stringBuilder = otherMap.entrySet().stream() - .map(entry -> String.format("%s=%s", entry.getKey(), entry.getValue())).collect(Collectors.toList()); - return String.join(";", stringBuilder); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split(";"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerConnectionParam.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerConnectionParam.java deleted file mode 100644 index 5ef72fdeaa..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerConnectionParam.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.sqlserver; - -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; - -public class SqlServerConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "SqlServerConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceProcessor.java deleted file mode 100644 index 5eef2cd49d..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceProcessor.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.sqlserver; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_SQLSERVER_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_SQLSERVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - SqlServerConnectionParam connectionParams = (SqlServerConnectionParam) createConnectionParams(connectionJson); - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - - SqlServerDatasourceParamDTO sqlServerDatasourceParamDTO = new SqlServerDatasourceParamDTO(); - sqlServerDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - sqlServerDatasourceParamDTO.setUserName(connectionParams.getUser()); - sqlServerDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - sqlServerDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - sqlServerDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - return sqlServerDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - SqlServerDatasourceParamDTO sqlServerParam = (SqlServerDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort()); - String jdbcUrl = address + ";databaseName=" + sqlServerParam.getDatabase(); - - SqlServerConnectionParam sqlServerConnectionParam = new SqlServerConnectionParam(); - sqlServerConnectionParam.setAddress(address); - sqlServerConnectionParam.setDatabase(sqlServerParam.getDatabase()); - sqlServerConnectionParam.setJdbcUrl(jdbcUrl); - sqlServerConnectionParam.setOther(transformOther(sqlServerParam.getOther())); - sqlServerConnectionParam.setUser(sqlServerParam.getUserName()); - sqlServerConnectionParam.setPassword(encodePassword(sqlServerParam.getPassword())); - return sqlServerConnectionParam; - } - - @Override - public BaseConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, SqlServerConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_SQLSERVER_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; - - if (StringUtils.isNotEmpty(sqlServerConnectionParam.getOther())) { - return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther()); - } - return sqlServerConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), sqlServerConnectionParam.getUser(), - decodePassword(sqlServerConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.SQLSERVER; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split(";")) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml index b75f26884c..dceeabc3d1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml @@ -39,6 +39,11 @@ dolphinscheduler-task-api ${project.version}
- * {@link ClickhouseConnectionParam} - * {@link Db2ConnectionParam} - * {@link HiveConnectionParam} - * {@link MysqlConnectionParam} - * {@link OracleConnectionParam} - * {@link PostgreSqlConnectionParam} - * {@link PrestoConnectionParam} - * {@link SparkConnectionParam} - * {@link SqlServerConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickhouseConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2ConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSqlConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkConnectionParam} + * {@link org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SqlServerConnectionParam} */ @JsonInclude(Include.NON_NULL) public abstract class BaseConnectionParam implements ConnectionParam { @@ -56,8 +49,16 @@ public abstract class BaseConnectionParam implements ConnectionParam { protected String jdbcUrl; + protected String driverLocation; + + protected String driverClassName; + + protected String validationQuery; + protected String other; + private Map props = new HashMap<>(); + public String getUser() { return user; } @@ -98,6 +99,30 @@ public abstract class BaseConnectionParam implements ConnectionParam { this.jdbcUrl = jdbcUrl; } + public String getDriverLocation() { + return driverLocation; + } + + public void setDriverLocation(String driverLocation) { + this.driverLocation = driverLocation; + } + + public String getDriverClassName() { + return driverClassName; + } + + public void setDriverClassName(String driverClassName) { + this.driverClassName = driverClassName; + } + + public String getValidationQuery() { + return validationQuery; + } + + public void setValidationQuery(String validationQuery) { + this.validationQuery = validationQuery; + } + public String getOther() { return other; } @@ -105,4 +130,12 @@ public abstract class BaseConnectionParam implements ConnectionParam { public void setOther(String other) { this.other = other; } + + public Map getProps() { + return props; + } + + public void setProps(Map props) { + this.props = props; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/ConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/ConnectionParam.java similarity index 94% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/ConnectionParam.java rename to dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/ConnectionParam.java index d4ec697751..b3eb903dc7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/ConnectionParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/ConnectionParam.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.datasource; +package org.apache.dolphinscheduler.spi.datasource; import java.io.Serializable; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/ConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannel.java similarity index 80% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/ConnectionParam.java rename to dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannel.java index 46bd979a2d..5328f9599a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/ConnectionParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannel.java @@ -15,12 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.datasource; +package org.apache.dolphinscheduler.spi.datasource; -import java.io.Serializable; +public interface DataSourceChannel { -/** - * The model of Datasource Connection param - */ -public interface ConnectionParam extends Serializable { + DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam); } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java new file mode 100644 index 0000000000..c947c3a647 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.datasource; + +public interface DataSourceChannelFactory { + /** + * get datasource client + */ + DataSourceChannel create(); + + /** + * get registry component name + */ + String getName(); +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceClient.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceClient.java new file mode 100644 index 0000000000..879d198284 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceClient.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.datasource; + +import java.sql.Connection; + +public interface DataSourceClient { + + void checkClient(); + + void close(); + + Connection getConnection(); +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 0c88407a02..3809c52269 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -22,31 +22,38 @@ import static java.util.stream.Collectors.toMap; import java.util.Arrays; import java.util.Map; +import com.baomidou.mybatisplus.annotation.EnumValue; import com.google.common.base.Functions; public enum DbType { + MYSQL(0, "mysql"), + POSTGRESQL(1, "postgresql"), + HIVE(2, "hive"), + SPARK(3, "spark"), + CLICKHOUSE(4, "clickhouse"), + ORACLE(5, "oracle"), + SQLSERVER(6, "sqlserver"), + DB2(7, "db2"), + PRESTO(8, "presto"), + H2(9, "h2"); - MYSQL(0), - POSTGRESQL(1), - HIVE(2), - SPARK(3), - CLICKHOUSE(4), - ORACLE(5), - SQLSERVER(6), - DB2(7), - PRESTO(8), - H2(9); - - DbType(int code) { + @EnumValue + private final int code; + private final String descp; + + DbType(int code, String descp) { this.code = code; + this.descp = descp; } - private final int code; - public int getCode() { return code; } + public String getDescp() { + return descp; + } + private static final Map DB_TYPE_MAP = Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); @@ -56,4 +63,5 @@ public enum DbType { } return null; } + } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginNotFoundException.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginException.java similarity index 69% rename from dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginNotFoundException.java rename to dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginException.java index 2153299f17..4e7dcf3db7 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginNotFoundException.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/exception/PluginException.java @@ -17,15 +17,23 @@ package org.apache.dolphinscheduler.spi.exception; -public class PluginNotFoundException extends RuntimeException { +public class PluginException extends RuntimeException { private static final long serialVersionUID = -5487812425126112159L; - public PluginNotFoundException(String message, Throwable cause) { + public PluginException(String message, Throwable cause) { super(message, cause); } - public PluginNotFoundException(String message) { + public PluginException(String message) { super(message); } + + public static PluginException getInstance(String message) { + return new PluginException(message); + } + + public static PluginException getInstance(String message, Throwable cause) { + return new PluginException(message + ": " + cause.getMessage(), cause); + } } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java index a3a8b77564..b2eb17bd54 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java @@ -72,4 +72,140 @@ public class Constants { */ public static final String YYYYMMDDHHMMSSSSS = "yyyyMMddHHmmssSSS"; + public static final String SPRING_DATASOURCE_MIN_IDLE = "spring.datasource.minIdle"; + + public static final String SPRING_DATASOURCE_MAX_ACTIVE = "spring.datasource.maxActive"; + + public static final String SPRING_DATASOURCE_TEST_ON_BORROW = "spring.datasource.testOnBorrow"; + + /** + * datasource plugin conf + */ + + public static final String DATASOURCE_PLUGIN_BINDING = "datasource.plugin.binding"; + + public static final String DATASOURCE_PLUGIN_DIR = "datasource.plugin.dir"; + + public static final String MAVEN_LOCAL_REPOSITORY = "maven.local.repository"; + + /** + * default datasource plugin dir + */ + public static final String DATASOURCE_PLUGIN_PATH = "lib/plugin/datasource"; + + /** + * java.security.krb5.conf + */ + public static final String JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + + /** + * java.security.krb5.conf.path + */ + public static final String JAVA_SECURITY_KRB5_CONF_PATH = "java.security.krb5.conf.path"; + + /** + * hadoop.security.authentication + */ + public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; + + /** + * hadoop.security.authentication + */ + public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state"; + + /** + * loginUserFromKeytab user + */ + public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username"; + + /** + * loginUserFromKeytab path + */ + public static final String LOGIN_USER_KEY_TAB_PATH = "login.user.keytab.path"; + + /** + * resource storage type + */ + public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type"; + + /** + * kerberos + */ + public static final String KERBEROS = "kerberos"; + + /** + * driver + */ + public static final String ORG_POSTGRESQL_DRIVER = "org.postgresql.Driver"; + public static final String COM_MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver"; + public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"; + public static final String COM_CLICKHOUSE_JDBC_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; + public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.OracleDriver"; + public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; + public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"; + + + /** + * validation Query + */ + public static final String POSTGRESQL_VALIDATION_QUERY = "select version()"; + public static final String MYSQL_VALIDATION_QUERY = "select 1"; + public static final String HIVE_VALIDATION_QUERY = "select 1"; + public static final String CLICKHOUSE_VALIDATION_QUERY = "select 1"; + public static final String ORACLE_VALIDATION_QUERY = "select 1 from dual"; + public static final String SQLSERVER_VALIDATION_QUERY = "select 1"; + public static final String DB2_VALIDATION_QUERY = "select 1 from sysibm.sysdummy1"; + public static final String PRESTO_VALIDATION_QUERY = "select 1"; + + /** + * jdbc url + */ + public static final String JDBC_MYSQL = "jdbc:mysql://"; + public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; + public static final String JDBC_HIVE_2 = "jdbc:hive2://"; + public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://"; + public static final String JDBC_ORACLE_SID = "jdbc:oracle:thin:@"; + public static final String JDBC_ORACLE_SERVICE_NAME = "jdbc:oracle:thin:@//"; + public static final String JDBC_SQLSERVER = "jdbc:sqlserver://"; + public static final String JDBC_DB2 = "jdbc:db2://"; + public static final String JDBC_PRESTO = "jdbc:presto://"; + + + public static final String ADDRESS = "address"; + public static final String DATABASE = "database"; + public static final String JDBC_URL = "jdbcUrl"; + public static final String PRINCIPAL = "principal"; + public static final String OTHER = "other"; + public static final String ORACLE_DB_CONNECT_TYPE = "connectType"; + public static final String KERBEROS_KRB5_CONF_PATH = "javaSecurityKrb5Conf"; + public static final String KERBEROS_KEY_TAB_USERNAME = "loginUserKeytabUsername"; + public static final String KERBEROS_KEY_TAB_PATH = "loginUserKeytabPath"; + + /** + * DOUBLE_SLASH // + */ + public static final String DOUBLE_SLASH = "//"; + + /** + * comma , + */ + public static final String COMMA = ","; + + /** + * COLON : + */ + public static final String COLON = ":"; + + /** + * AT SIGN + */ + public static final String AT_SIGN = "@"; + + /** + * datasource encryption salt + */ + public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*"; + public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable"; + public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt"; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/AbstractDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/AbstractDatasourceProcessor.java deleted file mode 100644 index 19d88e1588..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/AbstractDatasourceProcessor.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.commons.collections.MapUtils; - -import java.util.Map; -import java.util.regex.Pattern; - -public abstract class AbstractDatasourceProcessor implements DatasourceProcessor { - - private static final Pattern IPV4_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); - - private static final Pattern IPV6_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.\\:\\[\\]]+$"); - - private static final Pattern DATABASE_PATTER = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); - - private static final Pattern PARAMS_PATTER = Pattern.compile("^[a-zA-Z0-9\\-\\_\\/]+$"); - - @Override - public void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { - checkHost(baseDataSourceParamDTO.getHost()); - checkDatasourcePatter(baseDataSourceParamDTO.getDatabase()); - checkOther(baseDataSourceParamDTO.getOther()); - } - - /** - * Check the host is valid - * - * @param host datasource host - */ - protected void checkHost(String host) { - if (!IPV4_PATTERN.matcher(host).matches() || !IPV6_PATTERN.matcher(host).matches()) { - throw new IllegalArgumentException("datasource host illegal"); - } - } - - /** - * check database name is valid - * - * @param database database name - */ - protected void checkDatasourcePatter(String database) { - if (!DATABASE_PATTER.matcher(database).matches()) { - throw new IllegalArgumentException("datasource name illegal"); - } - } - - /** - * check other is valid - * - * @param other other - */ - protected void checkOther(Map other) { - if (MapUtils.isEmpty(other)) { - return; - } - boolean paramsCheck = other.entrySet().stream().allMatch(p -> PARAMS_PATTER.matcher(p.getValue()).matches()); - if (!paramsCheck) { - throw new IllegalArgumentException("datasource other params illegal"); - } - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseDataSourceParamDTO.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseDataSourceParamDTO.java deleted file mode 100644 index 5af330d38e..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseDataSourceParamDTO.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.dolphinscheduler.plugin.task.datasource.clickhouse.ClickHouseDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.db2.Db2DatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.hive.HiveDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.mysql.MysqlDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.oracle.OracleDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.postgresql.PostgreSqlDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.presto.PrestoDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.spark.SparkDatasourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.sqlserver.SqlServerDatasourceParamDTO; -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.io.Serializable; -import java.util.Map; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -/** - * Basic datasource params submitted to api. - * - * see {@link MysqlDatasourceParamDTO} - * see {@link PostgreSqlDatasourceParamDTO} - * see {@link HiveDataSourceParamDTO} - * see {@link SparkDatasourceParamDTO} - * see {@link ClickHouseDatasourceParamDTO} - * see {@link OracleDatasourceParamDTO} - * see {@link SqlServerDatasourceParamDTO} - * see {@link Db2DatasourceParamDTO} - * see {@link PrestoDatasourceParamDTO} - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -@JsonSubTypes(value = { - @JsonSubTypes.Type(value = MysqlDatasourceParamDTO.class, name = "MYSQL"), - @JsonSubTypes.Type(value = PostgreSqlDatasourceParamDTO.class, name = "POSTGRESQL"), - @JsonSubTypes.Type(value = HiveDataSourceParamDTO.class, name = "HIVE"), - @JsonSubTypes.Type(value = SparkDatasourceParamDTO.class, name = "SPARK"), - @JsonSubTypes.Type(value = ClickHouseDatasourceParamDTO.class, name = "CLICKHOUSE"), - @JsonSubTypes.Type(value = OracleDatasourceParamDTO.class, name = "ORACLE"), - @JsonSubTypes.Type(value = SqlServerDatasourceParamDTO.class, name = "SQLSERVER"), - @JsonSubTypes.Type(value = Db2DatasourceParamDTO.class, name = "DB2"), - @JsonSubTypes.Type(value = PrestoDatasourceParamDTO.class, name = "PRESTO"), -}) -public abstract class BaseDataSourceParamDTO implements Serializable { - - protected Integer id; - - protected String name; - - protected String note; - - protected String host; - - protected Integer port; - - protected String database; - - protected String userName; - - protected String password; - - protected Map other; - - public Integer getId() { - return id; - } - - public void setId(Integer id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getNote() { - return note; - } - - public void setNote(String note) { - this.note = note; - } - - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host; - } - - public Integer getPort() { - return port; - } - - public void setPort(Integer port) { - this.port = port; - } - - public String getDatabase() { - return database; - } - - public void setDatabase(String database) { - this.database = database; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public Map getOther() { - return other; - } - - public void setOther(Map other) { - this.other = other; - } - - /** - * Get the datasource type - * see{@link DbType} - * - * @return datasource type code - */ - public abstract DbType getType(); -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceUtil.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceUtil.java deleted file mode 100644 index d7b47f07ac..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceUtil.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.dolphinscheduler.plugin.task.datasource.clickhouse.ClickHouseDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.db2.Db2DatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.hive.HiveDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.mysql.MysqlDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.oracle.OracleDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.postgresql.PostgreSqlDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.presto.PrestoDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.spark.SparkDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.sqlserver.SqlServerDatasourceProcessor; -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.sql.Connection; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DatasourceUtil { - - private DatasourceUtil() { - } - - private static final Logger logger = LoggerFactory.getLogger(DatasourceUtil.class); - - private static final DatasourceProcessor mysqlProcessor = new MysqlDatasourceProcessor(); - private static final DatasourceProcessor postgreSqlProcessor = new PostgreSqlDatasourceProcessor(); - private static final DatasourceProcessor hiveProcessor = new HiveDatasourceProcessor(); - private static final DatasourceProcessor sparkProcessor = new SparkDatasourceProcessor(); - private static final DatasourceProcessor clickhouseProcessor = new ClickHouseDatasourceProcessor(); - private static final DatasourceProcessor oracleProcessor = new OracleDatasourceProcessor(); - private static final DatasourceProcessor sqlServerProcessor = new SqlServerDatasourceProcessor(); - private static final DatasourceProcessor db2PROCESSOR = new Db2DatasourceProcessor(); - private static final DatasourceProcessor prestoPROCESSOR = new PrestoDatasourceProcessor(); - - public static ConnectionParam buildConnectionParams(DbType dbType, String connectionJson) { - return getDatasourceProcessor(dbType).createConnectionParams(connectionJson); - } - - public static Connection getConnection(DbType dbType, ConnectionParam connectionParam) { - try { - return getDatasourceProcessor(dbType).getConnection(connectionParam); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static String getJdbcUrl(DbType dbType, ConnectionParam baseConnectionParam) { - return getDatasourceProcessor(dbType).getJdbcUrl(baseConnectionParam); - } - - public static DatasourceProcessor getDatasourceProcessor(DbType dbType) { - switch (dbType) { - case MYSQL: - return mysqlProcessor; - case POSTGRESQL: - return postgreSqlProcessor; - case HIVE: - return hiveProcessor; - case SPARK: - return sparkProcessor; - case CLICKHOUSE: - return clickhouseProcessor; - case ORACLE: - return oracleProcessor; - case SQLSERVER: - return sqlServerProcessor; - case DB2: - return db2PROCESSOR; - case PRESTO: - return prestoPROCESSOR; - default: - throw new IllegalArgumentException("datasource type illegal:" + dbType); - } - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/HiveConfUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/HiveConfUtils.java deleted file mode 100644 index 0985b7457c..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/HiveConfUtils.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * hive conf utils - */ -public class HiveConfUtils { - - private HiveConfUtils() { - throw new UnsupportedOperationException("Construct HiveConfUtils"); - } - - private static class HiveConfHandler { - private static HiveConf singleton; - - private static Map hiveConfVars; - - static { - singleton = new HiveConf(); - hiveConfVars = new HashMap<>(); - Arrays.stream(ConfVars.values()).forEach(confVar -> hiveConfVars.put(confVar.varname,confVar)); - } - } - - /** - * get HiveConf instance - * @return HiveConf hiveConf - */ - public static HiveConf getInstance() { - return HiveConfHandler.singleton; - } - - /** - * get hive conf vars - * @return - */ - public static Map getHiveConfVars() { - return HiveConfHandler.hiveConfVars; - } - - /** - * Determine if it belongs to a hive conf property - * @param conf config - * @return boolean result - */ - public static boolean isHiveConfVar(String conf) { - // the default hive conf var name - String confKey = conf.split("=")[0]; - Map hiveConfVars = HiveConfUtils.getHiveConfVars(); - if (hiveConfVars.get(confKey) != null) { - return true; - } - - // the security authorization hive conf var name - HiveConf hiveConf = HiveConfUtils.getInstance(); - String hiveAuthorizationSqlStdAuthConfigWhitelist = hiveConf.getVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST); - Pattern modWhiteListPattern = Pattern.compile(hiveAuthorizationSqlStdAuthConfigWhitelist); - Matcher matcher = modWhiteListPattern.matcher(confKey); - return matcher.matches(); - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java deleted file mode 100644 index 624afee940..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.clickhouse; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_CLICKHOUSE_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_CLICKHOUSE; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) createConnectionParams(connectionJson); - - ClickHouseDatasourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDatasourceParamDTO(); - clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser()); - clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - clickHouseDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - clickHouseDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - - return clickHouseDatasourceParamDTO; - } - - @Override - public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - ClickHouseDatasourceParamDTO clickHouseParam = (ClickHouseDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_CLICKHOUSE, clickHouseParam.getHost(), clickHouseParam.getPort()); - String jdbcUrl = address + "/" + clickHouseParam.getDatabase(); - - ClickhouseConnectionParam clickhouseConnectionParam = new ClickhouseConnectionParam(); - clickhouseConnectionParam.setDatabase(clickHouseParam.getDatabase()); - clickhouseConnectionParam.setAddress(address); - clickhouseConnectionParam.setJdbcUrl(jdbcUrl); - clickhouseConnectionParam.setUser(clickHouseParam.getUserName()); - clickhouseConnectionParam.setPassword(encodePassword(clickHouseParam.getPassword())); - clickhouseConnectionParam.setOther(transformOther(clickHouseParam.getOther())); - return clickhouseConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, ClickhouseConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_CLICKHOUSE_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; - String jdbcUrl = clickhouseConnectionParam.getJdbcUrl(); - if (StringUtils.isNotEmpty(clickhouseConnectionParam.getOther())) { - jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther()); - } - return jdbcUrl; - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam), - clickhouseConnectionParam.getUser(), decodePassword(clickhouseConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.CLICKHOUSE; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, "&"))); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (other == null) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split("&"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceProcessor.java deleted file mode 100644 index ee130ebe60..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceProcessor.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.db2; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_DB2_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_DB2; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class Db2DatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - Db2ConnectionParam connectionParams = (Db2ConnectionParam) createConnectionParams(connectionJson); - - Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO(); - db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName()); - - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - db2DatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - db2DatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return db2DatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - Db2DatasourceParamDTO db2Param = (Db2DatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_DB2, db2Param.getHost(), db2Param.getPort()); - String jdbcUrl = String.format("%s/%s", address, db2Param.getDatabase()); - - Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam(); - db2ConnectionParam.setAddress(address); - db2ConnectionParam.setDatabase(db2Param.getDatabase()); - db2ConnectionParam.setJdbcUrl(jdbcUrl); - db2ConnectionParam.setUser(db2Param.getUserName()); - db2ConnectionParam.setPassword(encodePassword(db2Param.getPassword())); - db2ConnectionParam.setOther(transformOther(db2Param.getOther())); - - return db2ConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, Db2ConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_DB2_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(db2ConnectionParam.getOther())) { - return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther()); - } - return db2ConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(db2ConnectionParam), - db2ConnectionParam.getUser(), decodePassword(db2ConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.DB2; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, ";"))); - stringBuilder.deleteCharAt(stringBuilder.length() - 1); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (other == null) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split("&")) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDatasourceProcessor.java deleted file mode 100644 index 16c4ceb111..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDatasourceProcessor.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.hive; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_HIVE_2; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.HiveConfUtils; -import org.apache.dolphinscheduler.plugin.task.util.CommonUtils; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); - HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) createConnectionParams(connectionJson); - - hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase()); - hiveDataSourceParamDTO.setUserName(hiveConnectionParam.getUser()); - hiveDataSourceParamDTO.setOther(parseOther(hiveConnectionParam.getOther())); - hiveDataSourceParamDTO.setLoginUserKeytabUsername(hiveConnectionParam.getLoginUserKeytabUsername()); - hiveDataSourceParamDTO.setLoginUserKeytabPath(hiveConnectionParam.getLoginUserKeytabPath()); - hiveDataSourceParamDTO.setJavaSecurityKrb5Conf(hiveConnectionParam.getJavaSecurityKrb5Conf()); - - String[] tmpArray = hiveConnectionParam.getAddress().split(DOUBLE_SLASH); - StringBuilder hosts = new StringBuilder(); - String[] hostPortArray = tmpArray[tmpArray.length - 1].split(COMMA); - for (String hostPort : hostPortArray) { - hosts.append(hostPort.split(COLON)[0]).append(COMMA); - } - hosts.deleteCharAt(hosts.length() - 1); - hiveDataSourceParamDTO.setHost(hosts.toString()); - hiveDataSourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return hiveDataSourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - HiveDataSourceParamDTO hiveParam = (HiveDataSourceParamDTO) datasourceParam; - StringBuilder address = new StringBuilder(); - address.append(JDBC_HIVE_2); - for (String zkHost : hiveParam.getHost().split(",")) { - address.append(String.format("%s:%s,", zkHost, hiveParam.getPort())); - } - address.deleteCharAt(address.length() - 1); - String jdbcUrl = address.toString() + "/" + hiveParam.getDatabase(); - if (CommonUtils.getKerberosStartupState()) { - jdbcUrl += ";principal=" + hiveParam.getPrincipal(); - } - - HiveConnectionParam hiveConnectionParam = new HiveConnectionParam(); - hiveConnectionParam.setDatabase(hiveParam.getDatabase()); - hiveConnectionParam.setAddress(address.toString()); - hiveConnectionParam.setJdbcUrl(jdbcUrl); - hiveConnectionParam.setUser(hiveParam.getUserName()); - hiveConnectionParam.setPassword(encodePassword(hiveParam.getPassword())); - - if (CommonUtils.getKerberosStartupState()) { - hiveConnectionParam.setPrincipal(hiveParam.getPrincipal()); - hiveConnectionParam.setJavaSecurityKrb5Conf(hiveParam.getJavaSecurityKrb5Conf()); - hiveConnectionParam.setLoginUserKeytabPath(hiveParam.getLoginUserKeytabPath()); - hiveConnectionParam.setLoginUserKeytabUsername(hiveParam.getLoginUserKeytabUsername()); - } - hiveConnectionParam.setOther(transformOther(hiveParam.getOther())); - return hiveConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, HiveConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; - String jdbcUrl = hiveConnectionParam.getJdbcUrl(); - String otherParams = filterOther(hiveConnectionParam.getOther()); - if (StringUtils.isNotEmpty(otherParams) && !"?".equals(otherParams.substring(0, 1))) { - jdbcUrl += ";"; - } - return jdbcUrl + otherParams; - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { - HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; - CommonUtils.loadKerberosConf(hiveConnectionParam.getJavaSecurityKrb5Conf(), - hiveConnectionParam.getLoginUserKeytabUsername(), hiveConnectionParam.getLoginUserKeytabPath()); - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), - hiveConnectionParam.getUser(), decodePassword(hiveConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.HIVE; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); - return stringBuilder.toString(); - } - - private String filterOther(String otherParams) { - if (StringUtils.isBlank(otherParams)) { - return ""; - } - - StringBuilder hiveConfListSb = new StringBuilder(); - hiveConfListSb.append("?"); - StringBuilder sessionVarListSb = new StringBuilder(); - - String[] otherArray = otherParams.split(";", -1); - - for (String conf : otherArray) { - if (HiveConfUtils.isHiveConfVar(conf)) { - hiveConfListSb.append(conf).append(";"); - } else { - sessionVarListSb.append(conf).append(";"); - } - } - - // remove the last ";" - if (sessionVarListSb.length() > 0) { - sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); - } - - if (hiveConfListSb.length() > 0) { - hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); - } - - return sessionVarListSb.toString() + hiveConfListSb.toString(); - } - - private Map parseOther(String other) { - if (other == null) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split(";"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceProcessor.java deleted file mode 100644 index e41ccd21c5..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceProcessor.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.mysql; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_MYSQL_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_MYSQL; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor { - - private final Logger logger = LoggerFactory.getLogger(MysqlDatasourceProcessor.class); - - private static final String ALLOW_LOAD_LOCAL_IN_FILE_NAME = "allowLoadLocalInfile"; - - private static final String AUTO_DESERIALIZE = "autoDeserialize"; - - private static final String ALLOW_LOCAL_IN_FILE_NAME = "allowLocalInfile"; - - private static final String ALLOW_URL_IN_LOCAL_IN_FILE_NAME = "allowUrlInLocalInfile"; - - private static final String APPEND_PARAMS = "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - MysqlConnectionParam connectionParams = (MysqlConnectionParam) createConnectionParams(connectionJson); - MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); - - mysqlDatasourceParamDTO.setUserName(connectionParams.getUser()); - mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - mysqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String address = connectionParams.getAddress(); - String[] hostSeperator = address.split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - mysqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - mysqlDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - - return mysqlDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { - MysqlDatasourceParamDTO mysqlDatasourceParam = (MysqlDatasourceParamDTO) dataSourceParam; - String address = String.format("%s%s:%s", JDBC_MYSQL, mysqlDatasourceParam.getHost(), mysqlDatasourceParam.getPort()); - String jdbcUrl = String.format("%s/%s", address, mysqlDatasourceParam.getDatabase()); - - MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); - mysqlConnectionParam.setJdbcUrl(jdbcUrl); - mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase()); - mysqlConnectionParam.setAddress(address); - mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName()); - mysqlConnectionParam.setPassword(encodePassword(mysqlDatasourceParam.getPassword())); - mysqlConnectionParam.setOther(transformOther(mysqlDatasourceParam.getOther())); - - return mysqlConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, MysqlConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_MYSQL_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; - String jdbcUrl = mysqlConnectionParam.getJdbcUrl(); - if (StringUtils.isNotEmpty(mysqlConnectionParam.getOther())) { - return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS); - } - return String.format("%s?%s", jdbcUrl, APPEND_PARAMS); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - String user = mysqlConnectionParam.getUser(); - if (user.contains(AUTO_DESERIALIZE)) { - logger.warn("sensitive param : {} in username field is filtered", AUTO_DESERIALIZE); - user = user.replace(AUTO_DESERIALIZE, ""); - } - String password = decodePassword(mysqlConnectionParam.getPassword()); - if (password.contains(AUTO_DESERIALIZE)) { - logger.warn("sensitive param : {} in password field is filtered", AUTO_DESERIALIZE); - password = password.replace(AUTO_DESERIALIZE, ""); - } - return DriverManager.getConnection(getJdbcUrl(connectionParam), user, password); - } - - @Override - public DbType getDbType() { - return DbType.MYSQL; - } - - private String transformOther(Map paramMap) { - if (MapUtils.isEmpty(paramMap)) { - return null; - } - Map otherMap = new HashMap<>(); - paramMap.forEach((k, v) -> { - if (!checkKeyIsLegitimate(k)) { - return; - } - otherMap.put(k, v); - }); - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); - return stringBuilder.toString(); - } - - private static boolean checkKeyIsLegitimate(String key) { - return !key.contains(ALLOW_LOAD_LOCAL_IN_FILE_NAME) - && !key.contains(AUTO_DESERIALIZE) - && !key.contains(ALLOW_LOCAL_IN_FILE_NAME) - && !key.contains(ALLOW_URL_IN_LOCAL_IN_FILE_NAME); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split("&")) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceProcessor.java deleted file mode 100644 index bfe94960b0..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceProcessor.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.oracle; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.AT_SIGN; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_ORACLE_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_ORACLE_SERVICE_NAME; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_ORACLE_SID; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbConnectType; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public class OracleDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - OracleConnectionParam connectionParams = (OracleConnectionParam) createConnectionParams(connectionJson); - OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO(); - - oracleDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - oracleDatasourceParamDTO.setUserName(connectionParams.getUser()); - oracleDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String hostSeperator = DOUBLE_SLASH; - if (DbConnectType.ORACLE_SID.equals(connectionParams.connectType)) { - hostSeperator = AT_SIGN; - } - String[] hostPort = connectionParams.getAddress().split(hostSeperator); - String[] hostPortArray = hostPort[hostPort.length - 1].split(COMMA); - oracleDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - oracleDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - - return oracleDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - OracleDatasourceParamDTO oracleParam = (OracleDatasourceParamDTO) datasourceParam; - String address; - if (DbConnectType.ORACLE_SID.equals(oracleParam.getConnectType())) { - address = String.format("%s%s:%s", - JDBC_ORACLE_SID, oracleParam.getHost(), oracleParam.getPort()); - } else { - address = String.format("%s%s:%s", - JDBC_ORACLE_SERVICE_NAME, oracleParam.getHost(), oracleParam.getPort()); - } - String jdbcUrl = address + "/" + oracleParam.getDatabase(); - - OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); - oracleConnectionParam.setUser(oracleParam.getUserName()); - oracleConnectionParam.setPassword(encodePassword(oracleParam.getPassword())); - oracleConnectionParam.setAddress(address); - oracleConnectionParam.setJdbcUrl(jdbcUrl); - oracleConnectionParam.setDatabase(oracleParam.getDatabase()); - oracleConnectionParam.setConnectType(oracleParam.getConnectType()); - oracleConnectionParam.setOther(transformOther(oracleParam.getOther())); - - return oracleConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, OracleConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_ORACLE_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(oracleConnectionParam.getOther())) { - return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther()); - } - return oracleConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), - oracleConnectionParam.getUser(), decodePassword(oracleConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.ORACLE; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - List list = new ArrayList<>(); - otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); - return String.join("&", list); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split("&"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlConnectionParam.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlConnectionParam.java deleted file mode 100644 index 1d459ca2b0..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlConnectionParam.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.postgresql; - -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; - -public class PostgreSqlConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "PostgreSqlConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java deleted file mode 100644 index fc705c6e41..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.postgresql; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_POSTGRESQL; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_POSTGRESQL_DRIVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) createConnectionParams(connectionJson); - PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO(); - postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser()); - postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String address = connectionParams.getAddress(); - String[] hostSeperator = address.split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - postgreSqlDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - postgreSqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return postgreSqlDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - PostgreSqlDatasourceParamDTO postgreSqlParam = (PostgreSqlDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_POSTGRESQL, postgreSqlParam.getHost(), postgreSqlParam.getPort()); - String jdbcUrl = String.format("%s/%s", address, postgreSqlParam.getDatabase()); - - PostgreSqlConnectionParam postgreSqlConnectionParam = new PostgreSqlConnectionParam(); - postgreSqlConnectionParam.setJdbcUrl(jdbcUrl); - postgreSqlConnectionParam.setAddress(address); - postgreSqlConnectionParam.setDatabase(postgreSqlParam.getDatabase()); - postgreSqlConnectionParam.setUser(postgreSqlParam.getUserName()); - postgreSqlConnectionParam.setPassword(encodePassword(postgreSqlParam.getPassword())); - postgreSqlConnectionParam.setOther(transformOther(postgreSqlParam.getOther())); - - return postgreSqlConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, PostgreSqlConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return ORG_POSTGRESQL_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(postgreSqlConnectionParam.getOther())) { - return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther()); - } - return postgreSqlConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam), - postgreSqlConnectionParam.getUser(), decodePassword(postgreSqlConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.POSTGRESQL; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split("&")) { - String[] split = config.split("="); - otherMap.put(split[0], split[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoConnectionParam.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoConnectionParam.java deleted file mode 100644 index b6a226d982..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoConnectionParam.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.presto; - -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; - -public class PrestoConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "PrestoConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceProcessor.java deleted file mode 100644 index d8630bf95f..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceProcessor.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.presto; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_PRESTO_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_PRESTO; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - PrestoConnectionParam connectionParams = (PrestoConnectionParam) createConnectionParams(connectionJson); - - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - - PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO(); - prestoDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - prestoDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - prestoDatasourceParamDTO.setUserName(connectionParams.getUser()); - prestoDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - return prestoDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - PrestoDatasourceParamDTO prestoParam = (PrestoDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_PRESTO, prestoParam.getHost(), prestoParam.getPort()); - String jdbcUrl = address + "/" + prestoParam.getDatabase(); - - PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam(); - prestoConnectionParam.setUser(prestoParam.getUserName()); - prestoConnectionParam.setPassword(encodePassword(prestoParam.getPassword())); - prestoConnectionParam.setOther(transformOther(prestoParam.getOther())); - prestoConnectionParam.setAddress(address); - prestoConnectionParam.setJdbcUrl(jdbcUrl); - prestoConnectionParam.setDatabase(prestoParam.getDatabase()); - - return prestoConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, PrestoConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_PRESTO_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(prestoConnectionParam.getOther())) { - return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther()); - } - return prestoConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), - prestoConnectionParam.getUser(), decodePassword(prestoConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.PRESTO; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isNotEmpty(otherMap)) { - List list = new ArrayList<>(); - otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); - return String.join("&", list); - } - return null; - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split("&"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceProcessor.java deleted file mode 100644 index 00e7ecbe0d..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceProcessor.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.spark; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_HIVE_2; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.plugin.task.util.CommonUtils; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class SparkDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - SparkConnectionParam connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson); - - SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO(); - sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - sparkDatasourceParamDTO.setUserName(connectionParams.getUser()); - sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - sparkDatasourceParamDTO.setJavaSecurityKrb5Conf(connectionParams.getJavaSecurityKrb5Conf()); - sparkDatasourceParamDTO.setLoginUserKeytabPath(connectionParams.getLoginUserKeytabPath()); - sparkDatasourceParamDTO.setLoginUserKeytabUsername(connectionParams.getLoginUserKeytabUsername()); - - StringBuilder hosts = new StringBuilder(); - String[] tmpArray = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = tmpArray[tmpArray.length - 1].split(COMMA); - Arrays.stream(hostPortArray).forEach(hostPort -> hosts.append(hostPort.split(COLON)[0]).append(COMMA)); - hosts.deleteCharAt(hosts.length() - 1); - - sparkDatasourceParamDTO.setHost(hosts.toString()); - sparkDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return sparkDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { - StringBuilder address = new StringBuilder(); - SparkDatasourceParamDTO sparkDatasourceParam = (SparkDatasourceParamDTO) dataSourceParam; - address.append(JDBC_HIVE_2); - for (String zkHost : sparkDatasourceParam.getHost().split(",")) { - address.append(String.format("%s:%s,", zkHost, sparkDatasourceParam.getPort())); - } - address.deleteCharAt(address.length() - 1); - - String jdbcUrl = address + "/" + sparkDatasourceParam.getDatabase(); - if (CommonUtils.getKerberosStartupState()) { - jdbcUrl += ";principal=" + sparkDatasourceParam.getPrincipal(); - } - - SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); - sparkConnectionParam.setPassword(encodePassword(sparkDatasourceParam.getPassword())); - sparkConnectionParam.setUser(sparkDatasourceParam.getUserName()); - sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther())); - sparkConnectionParam.setDatabase(sparkDatasourceParam.getDatabase()); - sparkConnectionParam.setAddress(address.toString()); - sparkConnectionParam.setJdbcUrl(jdbcUrl); - if (CommonUtils.getKerberosStartupState()) { - sparkConnectionParam.setPrincipal(sparkDatasourceParam.getPrincipal()); - sparkConnectionParam.setJavaSecurityKrb5Conf(sparkDatasourceParam.getJavaSecurityKrb5Conf()); - sparkConnectionParam.setLoginUserKeytabPath(sparkDatasourceParam.getLoginUserKeytabPath()); - sparkConnectionParam.setLoginUserKeytabUsername(sparkDatasourceParam.getLoginUserKeytabUsername()); - } - - return sparkConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, SparkConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(sparkConnectionParam.getOther())) { - return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther()); - } - return sparkConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { - SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; - CommonUtils.loadKerberosConf(sparkConnectionParam.getJavaSecurityKrb5Conf(), - sparkConnectionParam.getLoginUserKeytabUsername(), sparkConnectionParam.getLoginUserKeytabPath()); - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(sparkConnectionParam), - sparkConnectionParam.getUser(), decodePassword(sparkConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.SPARK; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - List stringBuilder = otherMap.entrySet().stream() - .map(entry -> String.format("%s=%s", entry.getKey(), entry.getValue())).collect(Collectors.toList()); - return String.join(";", stringBuilder); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split(";"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerConnectionParam.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerConnectionParam.java deleted file mode 100644 index 5ef72fdeaa..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerConnectionParam.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.sqlserver; - -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; - -public class SqlServerConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "SqlServerConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceProcessor.java deleted file mode 100644 index 5eef2cd49d..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceProcessor.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.sqlserver; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_SQLSERVER_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_SQLSERVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - SqlServerConnectionParam connectionParams = (SqlServerConnectionParam) createConnectionParams(connectionJson); - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - - SqlServerDatasourceParamDTO sqlServerDatasourceParamDTO = new SqlServerDatasourceParamDTO(); - sqlServerDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - sqlServerDatasourceParamDTO.setUserName(connectionParams.getUser()); - sqlServerDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - sqlServerDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - sqlServerDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - return sqlServerDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - SqlServerDatasourceParamDTO sqlServerParam = (SqlServerDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort()); - String jdbcUrl = address + ";databaseName=" + sqlServerParam.getDatabase(); - - SqlServerConnectionParam sqlServerConnectionParam = new SqlServerConnectionParam(); - sqlServerConnectionParam.setAddress(address); - sqlServerConnectionParam.setDatabase(sqlServerParam.getDatabase()); - sqlServerConnectionParam.setJdbcUrl(jdbcUrl); - sqlServerConnectionParam.setOther(transformOther(sqlServerParam.getOther())); - sqlServerConnectionParam.setUser(sqlServerParam.getUserName()); - sqlServerConnectionParam.setPassword(encodePassword(sqlServerParam.getPassword())); - return sqlServerConnectionParam; - } - - @Override - public BaseConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, SqlServerConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_SQLSERVER_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; - - if (StringUtils.isNotEmpty(sqlServerConnectionParam.getOther())) { - return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther()); - } - return sqlServerConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), sqlServerConnectionParam.getUser(), - decodePassword(sqlServerConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.SQLSERVER; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split(";")) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml index b75f26884c..dceeabc3d1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml @@ -39,6 +39,11 @@ dolphinscheduler-task-api ${project.version}
- * see {@link MysqlDatasourceParamDTO} - * see {@link PostgreSqlDatasourceParamDTO} - * see {@link HiveDataSourceParamDTO} - * see {@link SparkDatasourceParamDTO} - * see {@link ClickHouseDatasourceParamDTO} - * see {@link OracleDatasourceParamDTO} - * see {@link SqlServerDatasourceParamDTO} - * see {@link Db2DatasourceParamDTO} - * see {@link PrestoDatasourceParamDTO} - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -@JsonSubTypes(value = { - @JsonSubTypes.Type(value = MysqlDatasourceParamDTO.class, name = "MYSQL"), - @JsonSubTypes.Type(value = PostgreSqlDatasourceParamDTO.class, name = "POSTGRESQL"), - @JsonSubTypes.Type(value = HiveDataSourceParamDTO.class, name = "HIVE"), - @JsonSubTypes.Type(value = SparkDatasourceParamDTO.class, name = "SPARK"), - @JsonSubTypes.Type(value = ClickHouseDatasourceParamDTO.class, name = "CLICKHOUSE"), - @JsonSubTypes.Type(value = OracleDatasourceParamDTO.class, name = "ORACLE"), - @JsonSubTypes.Type(value = SqlServerDatasourceParamDTO.class, name = "SQLSERVER"), - @JsonSubTypes.Type(value = Db2DatasourceParamDTO.class, name = "DB2"), - @JsonSubTypes.Type(value = PrestoDatasourceParamDTO.class, name = "PRESTO"), -}) -public abstract class BaseDataSourceParamDTO implements Serializable { - - protected Integer id; - - protected String name; - - protected String note; - - protected String host; - - protected Integer port; - - protected String database; - - protected String userName; - - protected String password; - - protected Map other; - - public Integer getId() { - return id; - } - - public void setId(Integer id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getNote() { - return note; - } - - public void setNote(String note) { - this.note = note; - } - - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host; - } - - public Integer getPort() { - return port; - } - - public void setPort(Integer port) { - this.port = port; - } - - public String getDatabase() { - return database; - } - - public void setDatabase(String database) { - this.database = database; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public Map getOther() { - return other; - } - - public void setOther(Map other) { - this.other = other; - } - - /** - * Get the datasource type - * see{@link DbType} - * - * @return datasource type code - */ - public abstract DbType getType(); -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceUtil.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceUtil.java deleted file mode 100644 index d7b47f07ac..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceUtil.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.dolphinscheduler.plugin.task.datasource.clickhouse.ClickHouseDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.db2.Db2DatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.hive.HiveDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.mysql.MysqlDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.oracle.OracleDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.postgresql.PostgreSqlDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.presto.PrestoDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.spark.SparkDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.sqlserver.SqlServerDatasourceProcessor; -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.sql.Connection; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DatasourceUtil { - - private DatasourceUtil() { - } - - private static final Logger logger = LoggerFactory.getLogger(DatasourceUtil.class); - - private static final DatasourceProcessor mysqlProcessor = new MysqlDatasourceProcessor(); - private static final DatasourceProcessor postgreSqlProcessor = new PostgreSqlDatasourceProcessor(); - private static final DatasourceProcessor hiveProcessor = new HiveDatasourceProcessor(); - private static final DatasourceProcessor sparkProcessor = new SparkDatasourceProcessor(); - private static final DatasourceProcessor clickhouseProcessor = new ClickHouseDatasourceProcessor(); - private static final DatasourceProcessor oracleProcessor = new OracleDatasourceProcessor(); - private static final DatasourceProcessor sqlServerProcessor = new SqlServerDatasourceProcessor(); - private static final DatasourceProcessor db2PROCESSOR = new Db2DatasourceProcessor(); - private static final DatasourceProcessor prestoPROCESSOR = new PrestoDatasourceProcessor(); - - public static ConnectionParam buildConnectionParams(DbType dbType, String connectionJson) { - return getDatasourceProcessor(dbType).createConnectionParams(connectionJson); - } - - public static Connection getConnection(DbType dbType, ConnectionParam connectionParam) { - try { - return getDatasourceProcessor(dbType).getConnection(connectionParam); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static String getJdbcUrl(DbType dbType, ConnectionParam baseConnectionParam) { - return getDatasourceProcessor(dbType).getJdbcUrl(baseConnectionParam); - } - - public static DatasourceProcessor getDatasourceProcessor(DbType dbType) { - switch (dbType) { - case MYSQL: - return mysqlProcessor; - case POSTGRESQL: - return postgreSqlProcessor; - case HIVE: - return hiveProcessor; - case SPARK: - return sparkProcessor; - case CLICKHOUSE: - return clickhouseProcessor; - case ORACLE: - return oracleProcessor; - case SQLSERVER: - return sqlServerProcessor; - case DB2: - return db2PROCESSOR; - case PRESTO: - return prestoPROCESSOR; - default: - throw new IllegalArgumentException("datasource type illegal:" + dbType); - } - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/HiveConfUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/HiveConfUtils.java deleted file mode 100644 index 0985b7457c..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/HiveConfUtils.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * hive conf utils - */ -public class HiveConfUtils { - - private HiveConfUtils() { - throw new UnsupportedOperationException("Construct HiveConfUtils"); - } - - private static class HiveConfHandler { - private static HiveConf singleton; - - private static Map hiveConfVars; - - static { - singleton = new HiveConf(); - hiveConfVars = new HashMap<>(); - Arrays.stream(ConfVars.values()).forEach(confVar -> hiveConfVars.put(confVar.varname,confVar)); - } - } - - /** - * get HiveConf instance - * @return HiveConf hiveConf - */ - public static HiveConf getInstance() { - return HiveConfHandler.singleton; - } - - /** - * get hive conf vars - * @return - */ - public static Map getHiveConfVars() { - return HiveConfHandler.hiveConfVars; - } - - /** - * Determine if it belongs to a hive conf property - * @param conf config - * @return boolean result - */ - public static boolean isHiveConfVar(String conf) { - // the default hive conf var name - String confKey = conf.split("=")[0]; - Map hiveConfVars = HiveConfUtils.getHiveConfVars(); - if (hiveConfVars.get(confKey) != null) { - return true; - } - - // the security authorization hive conf var name - HiveConf hiveConf = HiveConfUtils.getInstance(); - String hiveAuthorizationSqlStdAuthConfigWhitelist = hiveConf.getVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST); - Pattern modWhiteListPattern = Pattern.compile(hiveAuthorizationSqlStdAuthConfigWhitelist); - Matcher matcher = modWhiteListPattern.matcher(confKey); - return matcher.matches(); - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java deleted file mode 100644 index 624afee940..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.clickhouse; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_CLICKHOUSE_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_CLICKHOUSE; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) createConnectionParams(connectionJson); - - ClickHouseDatasourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDatasourceParamDTO(); - clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser()); - clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - clickHouseDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - clickHouseDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - - return clickHouseDatasourceParamDTO; - } - - @Override - public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - ClickHouseDatasourceParamDTO clickHouseParam = (ClickHouseDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_CLICKHOUSE, clickHouseParam.getHost(), clickHouseParam.getPort()); - String jdbcUrl = address + "/" + clickHouseParam.getDatabase(); - - ClickhouseConnectionParam clickhouseConnectionParam = new ClickhouseConnectionParam(); - clickhouseConnectionParam.setDatabase(clickHouseParam.getDatabase()); - clickhouseConnectionParam.setAddress(address); - clickhouseConnectionParam.setJdbcUrl(jdbcUrl); - clickhouseConnectionParam.setUser(clickHouseParam.getUserName()); - clickhouseConnectionParam.setPassword(encodePassword(clickHouseParam.getPassword())); - clickhouseConnectionParam.setOther(transformOther(clickHouseParam.getOther())); - return clickhouseConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, ClickhouseConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_CLICKHOUSE_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; - String jdbcUrl = clickhouseConnectionParam.getJdbcUrl(); - if (StringUtils.isNotEmpty(clickhouseConnectionParam.getOther())) { - jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther()); - } - return jdbcUrl; - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam), - clickhouseConnectionParam.getUser(), decodePassword(clickhouseConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.CLICKHOUSE; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, "&"))); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (other == null) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split("&"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceProcessor.java deleted file mode 100644 index ee130ebe60..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceProcessor.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.db2; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_DB2_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_DB2; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class Db2DatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - Db2ConnectionParam connectionParams = (Db2ConnectionParam) createConnectionParams(connectionJson); - - Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO(); - db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName()); - - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - db2DatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - db2DatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return db2DatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - Db2DatasourceParamDTO db2Param = (Db2DatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_DB2, db2Param.getHost(), db2Param.getPort()); - String jdbcUrl = String.format("%s/%s", address, db2Param.getDatabase()); - - Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam(); - db2ConnectionParam.setAddress(address); - db2ConnectionParam.setDatabase(db2Param.getDatabase()); - db2ConnectionParam.setJdbcUrl(jdbcUrl); - db2ConnectionParam.setUser(db2Param.getUserName()); - db2ConnectionParam.setPassword(encodePassword(db2Param.getPassword())); - db2ConnectionParam.setOther(transformOther(db2Param.getOther())); - - return db2ConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, Db2ConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_DB2_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(db2ConnectionParam.getOther())) { - return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther()); - } - return db2ConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(db2ConnectionParam), - db2ConnectionParam.getUser(), decodePassword(db2ConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.DB2; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, ";"))); - stringBuilder.deleteCharAt(stringBuilder.length() - 1); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (other == null) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split("&")) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDatasourceProcessor.java deleted file mode 100644 index 16c4ceb111..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDatasourceProcessor.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.hive; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_HIVE_2; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.HiveConfUtils; -import org.apache.dolphinscheduler.plugin.task.util.CommonUtils; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); - HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) createConnectionParams(connectionJson); - - hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase()); - hiveDataSourceParamDTO.setUserName(hiveConnectionParam.getUser()); - hiveDataSourceParamDTO.setOther(parseOther(hiveConnectionParam.getOther())); - hiveDataSourceParamDTO.setLoginUserKeytabUsername(hiveConnectionParam.getLoginUserKeytabUsername()); - hiveDataSourceParamDTO.setLoginUserKeytabPath(hiveConnectionParam.getLoginUserKeytabPath()); - hiveDataSourceParamDTO.setJavaSecurityKrb5Conf(hiveConnectionParam.getJavaSecurityKrb5Conf()); - - String[] tmpArray = hiveConnectionParam.getAddress().split(DOUBLE_SLASH); - StringBuilder hosts = new StringBuilder(); - String[] hostPortArray = tmpArray[tmpArray.length - 1].split(COMMA); - for (String hostPort : hostPortArray) { - hosts.append(hostPort.split(COLON)[0]).append(COMMA); - } - hosts.deleteCharAt(hosts.length() - 1); - hiveDataSourceParamDTO.setHost(hosts.toString()); - hiveDataSourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return hiveDataSourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - HiveDataSourceParamDTO hiveParam = (HiveDataSourceParamDTO) datasourceParam; - StringBuilder address = new StringBuilder(); - address.append(JDBC_HIVE_2); - for (String zkHost : hiveParam.getHost().split(",")) { - address.append(String.format("%s:%s,", zkHost, hiveParam.getPort())); - } - address.deleteCharAt(address.length() - 1); - String jdbcUrl = address.toString() + "/" + hiveParam.getDatabase(); - if (CommonUtils.getKerberosStartupState()) { - jdbcUrl += ";principal=" + hiveParam.getPrincipal(); - } - - HiveConnectionParam hiveConnectionParam = new HiveConnectionParam(); - hiveConnectionParam.setDatabase(hiveParam.getDatabase()); - hiveConnectionParam.setAddress(address.toString()); - hiveConnectionParam.setJdbcUrl(jdbcUrl); - hiveConnectionParam.setUser(hiveParam.getUserName()); - hiveConnectionParam.setPassword(encodePassword(hiveParam.getPassword())); - - if (CommonUtils.getKerberosStartupState()) { - hiveConnectionParam.setPrincipal(hiveParam.getPrincipal()); - hiveConnectionParam.setJavaSecurityKrb5Conf(hiveParam.getJavaSecurityKrb5Conf()); - hiveConnectionParam.setLoginUserKeytabPath(hiveParam.getLoginUserKeytabPath()); - hiveConnectionParam.setLoginUserKeytabUsername(hiveParam.getLoginUserKeytabUsername()); - } - hiveConnectionParam.setOther(transformOther(hiveParam.getOther())); - return hiveConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, HiveConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; - String jdbcUrl = hiveConnectionParam.getJdbcUrl(); - String otherParams = filterOther(hiveConnectionParam.getOther()); - if (StringUtils.isNotEmpty(otherParams) && !"?".equals(otherParams.substring(0, 1))) { - jdbcUrl += ";"; - } - return jdbcUrl + otherParams; - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { - HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; - CommonUtils.loadKerberosConf(hiveConnectionParam.getJavaSecurityKrb5Conf(), - hiveConnectionParam.getLoginUserKeytabUsername(), hiveConnectionParam.getLoginUserKeytabPath()); - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), - hiveConnectionParam.getUser(), decodePassword(hiveConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.HIVE; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); - return stringBuilder.toString(); - } - - private String filterOther(String otherParams) { - if (StringUtils.isBlank(otherParams)) { - return ""; - } - - StringBuilder hiveConfListSb = new StringBuilder(); - hiveConfListSb.append("?"); - StringBuilder sessionVarListSb = new StringBuilder(); - - String[] otherArray = otherParams.split(";", -1); - - for (String conf : otherArray) { - if (HiveConfUtils.isHiveConfVar(conf)) { - hiveConfListSb.append(conf).append(";"); - } else { - sessionVarListSb.append(conf).append(";"); - } - } - - // remove the last ";" - if (sessionVarListSb.length() > 0) { - sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); - } - - if (hiveConfListSb.length() > 0) { - hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); - } - - return sessionVarListSb.toString() + hiveConfListSb.toString(); - } - - private Map parseOther(String other) { - if (other == null) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split(";"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceProcessor.java deleted file mode 100644 index e41ccd21c5..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceProcessor.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.mysql; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_MYSQL_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_MYSQL; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor { - - private final Logger logger = LoggerFactory.getLogger(MysqlDatasourceProcessor.class); - - private static final String ALLOW_LOAD_LOCAL_IN_FILE_NAME = "allowLoadLocalInfile"; - - private static final String AUTO_DESERIALIZE = "autoDeserialize"; - - private static final String ALLOW_LOCAL_IN_FILE_NAME = "allowLocalInfile"; - - private static final String ALLOW_URL_IN_LOCAL_IN_FILE_NAME = "allowUrlInLocalInfile"; - - private static final String APPEND_PARAMS = "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - MysqlConnectionParam connectionParams = (MysqlConnectionParam) createConnectionParams(connectionJson); - MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); - - mysqlDatasourceParamDTO.setUserName(connectionParams.getUser()); - mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - mysqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String address = connectionParams.getAddress(); - String[] hostSeperator = address.split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - mysqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - mysqlDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - - return mysqlDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { - MysqlDatasourceParamDTO mysqlDatasourceParam = (MysqlDatasourceParamDTO) dataSourceParam; - String address = String.format("%s%s:%s", JDBC_MYSQL, mysqlDatasourceParam.getHost(), mysqlDatasourceParam.getPort()); - String jdbcUrl = String.format("%s/%s", address, mysqlDatasourceParam.getDatabase()); - - MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); - mysqlConnectionParam.setJdbcUrl(jdbcUrl); - mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase()); - mysqlConnectionParam.setAddress(address); - mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName()); - mysqlConnectionParam.setPassword(encodePassword(mysqlDatasourceParam.getPassword())); - mysqlConnectionParam.setOther(transformOther(mysqlDatasourceParam.getOther())); - - return mysqlConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, MysqlConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_MYSQL_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; - String jdbcUrl = mysqlConnectionParam.getJdbcUrl(); - if (StringUtils.isNotEmpty(mysqlConnectionParam.getOther())) { - return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS); - } - return String.format("%s?%s", jdbcUrl, APPEND_PARAMS); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - String user = mysqlConnectionParam.getUser(); - if (user.contains(AUTO_DESERIALIZE)) { - logger.warn("sensitive param : {} in username field is filtered", AUTO_DESERIALIZE); - user = user.replace(AUTO_DESERIALIZE, ""); - } - String password = decodePassword(mysqlConnectionParam.getPassword()); - if (password.contains(AUTO_DESERIALIZE)) { - logger.warn("sensitive param : {} in password field is filtered", AUTO_DESERIALIZE); - password = password.replace(AUTO_DESERIALIZE, ""); - } - return DriverManager.getConnection(getJdbcUrl(connectionParam), user, password); - } - - @Override - public DbType getDbType() { - return DbType.MYSQL; - } - - private String transformOther(Map paramMap) { - if (MapUtils.isEmpty(paramMap)) { - return null; - } - Map otherMap = new HashMap<>(); - paramMap.forEach((k, v) -> { - if (!checkKeyIsLegitimate(k)) { - return; - } - otherMap.put(k, v); - }); - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); - return stringBuilder.toString(); - } - - private static boolean checkKeyIsLegitimate(String key) { - return !key.contains(ALLOW_LOAD_LOCAL_IN_FILE_NAME) - && !key.contains(AUTO_DESERIALIZE) - && !key.contains(ALLOW_LOCAL_IN_FILE_NAME) - && !key.contains(ALLOW_URL_IN_LOCAL_IN_FILE_NAME); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split("&")) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceProcessor.java deleted file mode 100644 index bfe94960b0..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceProcessor.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.oracle; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.AT_SIGN; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_ORACLE_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_ORACLE_SERVICE_NAME; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_ORACLE_SID; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbConnectType; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public class OracleDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - OracleConnectionParam connectionParams = (OracleConnectionParam) createConnectionParams(connectionJson); - OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO(); - - oracleDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - oracleDatasourceParamDTO.setUserName(connectionParams.getUser()); - oracleDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String hostSeperator = DOUBLE_SLASH; - if (DbConnectType.ORACLE_SID.equals(connectionParams.connectType)) { - hostSeperator = AT_SIGN; - } - String[] hostPort = connectionParams.getAddress().split(hostSeperator); - String[] hostPortArray = hostPort[hostPort.length - 1].split(COMMA); - oracleDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - oracleDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - - return oracleDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - OracleDatasourceParamDTO oracleParam = (OracleDatasourceParamDTO) datasourceParam; - String address; - if (DbConnectType.ORACLE_SID.equals(oracleParam.getConnectType())) { - address = String.format("%s%s:%s", - JDBC_ORACLE_SID, oracleParam.getHost(), oracleParam.getPort()); - } else { - address = String.format("%s%s:%s", - JDBC_ORACLE_SERVICE_NAME, oracleParam.getHost(), oracleParam.getPort()); - } - String jdbcUrl = address + "/" + oracleParam.getDatabase(); - - OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); - oracleConnectionParam.setUser(oracleParam.getUserName()); - oracleConnectionParam.setPassword(encodePassword(oracleParam.getPassword())); - oracleConnectionParam.setAddress(address); - oracleConnectionParam.setJdbcUrl(jdbcUrl); - oracleConnectionParam.setDatabase(oracleParam.getDatabase()); - oracleConnectionParam.setConnectType(oracleParam.getConnectType()); - oracleConnectionParam.setOther(transformOther(oracleParam.getOther())); - - return oracleConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, OracleConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_ORACLE_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(oracleConnectionParam.getOther())) { - return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther()); - } - return oracleConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), - oracleConnectionParam.getUser(), decodePassword(oracleConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.ORACLE; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - List list = new ArrayList<>(); - otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); - return String.join("&", list); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split("&"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlConnectionParam.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlConnectionParam.java deleted file mode 100644 index 1d459ca2b0..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlConnectionParam.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.postgresql; - -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; - -public class PostgreSqlConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "PostgreSqlConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java deleted file mode 100644 index fc705c6e41..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.postgresql; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_POSTGRESQL; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_POSTGRESQL_DRIVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) createConnectionParams(connectionJson); - PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO(); - postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser()); - postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - String address = connectionParams.getAddress(); - String[] hostSeperator = address.split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - postgreSqlDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - postgreSqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return postgreSqlDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - PostgreSqlDatasourceParamDTO postgreSqlParam = (PostgreSqlDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_POSTGRESQL, postgreSqlParam.getHost(), postgreSqlParam.getPort()); - String jdbcUrl = String.format("%s/%s", address, postgreSqlParam.getDatabase()); - - PostgreSqlConnectionParam postgreSqlConnectionParam = new PostgreSqlConnectionParam(); - postgreSqlConnectionParam.setJdbcUrl(jdbcUrl); - postgreSqlConnectionParam.setAddress(address); - postgreSqlConnectionParam.setDatabase(postgreSqlParam.getDatabase()); - postgreSqlConnectionParam.setUser(postgreSqlParam.getUserName()); - postgreSqlConnectionParam.setPassword(encodePassword(postgreSqlParam.getPassword())); - postgreSqlConnectionParam.setOther(transformOther(postgreSqlParam.getOther())); - - return postgreSqlConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, PostgreSqlConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return ORG_POSTGRESQL_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(postgreSqlConnectionParam.getOther())) { - return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther()); - } - return postgreSqlConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam), - postgreSqlConnectionParam.getUser(), decodePassword(postgreSqlConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.POSTGRESQL; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split("&")) { - String[] split = config.split("="); - otherMap.put(split[0], split[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoConnectionParam.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoConnectionParam.java deleted file mode 100644 index b6a226d982..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoConnectionParam.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.presto; - -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; - -public class PrestoConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "PrestoConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceProcessor.java deleted file mode 100644 index d8630bf95f..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceProcessor.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.presto; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_PRESTO_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_PRESTO; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - PrestoConnectionParam connectionParams = (PrestoConnectionParam) createConnectionParams(connectionJson); - - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - - PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO(); - prestoDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - prestoDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - prestoDatasourceParamDTO.setUserName(connectionParams.getUser()); - prestoDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - - return prestoDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - PrestoDatasourceParamDTO prestoParam = (PrestoDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_PRESTO, prestoParam.getHost(), prestoParam.getPort()); - String jdbcUrl = address + "/" + prestoParam.getDatabase(); - - PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam(); - prestoConnectionParam.setUser(prestoParam.getUserName()); - prestoConnectionParam.setPassword(encodePassword(prestoParam.getPassword())); - prestoConnectionParam.setOther(transformOther(prestoParam.getOther())); - prestoConnectionParam.setAddress(address); - prestoConnectionParam.setJdbcUrl(jdbcUrl); - prestoConnectionParam.setDatabase(prestoParam.getDatabase()); - - return prestoConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, PrestoConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_PRESTO_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(prestoConnectionParam.getOther())) { - return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther()); - } - return prestoConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), - prestoConnectionParam.getUser(), decodePassword(prestoConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.PRESTO; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isNotEmpty(otherMap)) { - List list = new ArrayList<>(); - otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); - return String.join("&", list); - } - return null; - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split("&"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceProcessor.java deleted file mode 100644 index 00e7ecbe0d..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceProcessor.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.spark; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_HIVE_2; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.plugin.task.util.CommonUtils; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class SparkDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - SparkConnectionParam connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson); - - SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO(); - sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - sparkDatasourceParamDTO.setUserName(connectionParams.getUser()); - sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - sparkDatasourceParamDTO.setJavaSecurityKrb5Conf(connectionParams.getJavaSecurityKrb5Conf()); - sparkDatasourceParamDTO.setLoginUserKeytabPath(connectionParams.getLoginUserKeytabPath()); - sparkDatasourceParamDTO.setLoginUserKeytabUsername(connectionParams.getLoginUserKeytabUsername()); - - StringBuilder hosts = new StringBuilder(); - String[] tmpArray = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = tmpArray[tmpArray.length - 1].split(COMMA); - Arrays.stream(hostPortArray).forEach(hostPort -> hosts.append(hostPort.split(COLON)[0]).append(COMMA)); - hosts.deleteCharAt(hosts.length() - 1); - - sparkDatasourceParamDTO.setHost(hosts.toString()); - sparkDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - - return sparkDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { - StringBuilder address = new StringBuilder(); - SparkDatasourceParamDTO sparkDatasourceParam = (SparkDatasourceParamDTO) dataSourceParam; - address.append(JDBC_HIVE_2); - for (String zkHost : sparkDatasourceParam.getHost().split(",")) { - address.append(String.format("%s:%s,", zkHost, sparkDatasourceParam.getPort())); - } - address.deleteCharAt(address.length() - 1); - - String jdbcUrl = address + "/" + sparkDatasourceParam.getDatabase(); - if (CommonUtils.getKerberosStartupState()) { - jdbcUrl += ";principal=" + sparkDatasourceParam.getPrincipal(); - } - - SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); - sparkConnectionParam.setPassword(encodePassword(sparkDatasourceParam.getPassword())); - sparkConnectionParam.setUser(sparkDatasourceParam.getUserName()); - sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther())); - sparkConnectionParam.setDatabase(sparkDatasourceParam.getDatabase()); - sparkConnectionParam.setAddress(address.toString()); - sparkConnectionParam.setJdbcUrl(jdbcUrl); - if (CommonUtils.getKerberosStartupState()) { - sparkConnectionParam.setPrincipal(sparkDatasourceParam.getPrincipal()); - sparkConnectionParam.setJavaSecurityKrb5Conf(sparkDatasourceParam.getJavaSecurityKrb5Conf()); - sparkConnectionParam.setLoginUserKeytabPath(sparkDatasourceParam.getLoginUserKeytabPath()); - sparkConnectionParam.setLoginUserKeytabUsername(sparkDatasourceParam.getLoginUserKeytabUsername()); - } - - return sparkConnectionParam; - } - - @Override - public ConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, SparkConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; - if (StringUtils.isNotEmpty(sparkConnectionParam.getOther())) { - return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther()); - } - return sparkConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { - SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; - CommonUtils.loadKerberosConf(sparkConnectionParam.getJavaSecurityKrb5Conf(), - sparkConnectionParam.getLoginUserKeytabUsername(), sparkConnectionParam.getLoginUserKeytabPath()); - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(sparkConnectionParam), - sparkConnectionParam.getUser(), decodePassword(sparkConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.SPARK; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - List stringBuilder = otherMap.entrySet().stream() - .map(entry -> String.format("%s=%s", entry.getKey(), entry.getValue())).collect(Collectors.toList()); - return String.join(";", stringBuilder); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - String[] configs = other.split(";"); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerConnectionParam.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerConnectionParam.java deleted file mode 100644 index 5ef72fdeaa..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerConnectionParam.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.sqlserver; - -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; - -public class SqlServerConnectionParam extends BaseConnectionParam { - @Override - public String toString() { - return "SqlServerConnectionParam{" - + "user='" + user + '\'' - + ", password='" + password + '\'' - + ", address='" + address + '\'' - + ", database='" + database + '\'' - + ", jdbcUrl='" + jdbcUrl + '\'' - + ", other='" + other + '\'' - + '}'; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceProcessor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceProcessor.java deleted file mode 100644 index 5eef2cd49d..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceProcessor.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.datasource.sqlserver; - -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.datasource.PasswordUtils.encodePassword; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_SQLSERVER_JDBC_DRIVER; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; -import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_SQLSERVER; - -import org.apache.dolphinscheduler.plugin.task.datasource.AbstractDatasourceProcessor; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO; -import org.apache.dolphinscheduler.plugin.task.datasource.ConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.StringUtils; - -import org.apache.commons.collections.MapUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; - -public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor { - - @Override - public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - SqlServerConnectionParam connectionParams = (SqlServerConnectionParam) createConnectionParams(connectionJson); - String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); - String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); - - SqlServerDatasourceParamDTO sqlServerDatasourceParamDTO = new SqlServerDatasourceParamDTO(); - sqlServerDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); - sqlServerDatasourceParamDTO.setUserName(connectionParams.getUser()); - sqlServerDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); - sqlServerDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); - sqlServerDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); - return sqlServerDatasourceParamDTO; - } - - @Override - public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { - SqlServerDatasourceParamDTO sqlServerParam = (SqlServerDatasourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort()); - String jdbcUrl = address + ";databaseName=" + sqlServerParam.getDatabase(); - - SqlServerConnectionParam sqlServerConnectionParam = new SqlServerConnectionParam(); - sqlServerConnectionParam.setAddress(address); - sqlServerConnectionParam.setDatabase(sqlServerParam.getDatabase()); - sqlServerConnectionParam.setJdbcUrl(jdbcUrl); - sqlServerConnectionParam.setOther(transformOther(sqlServerParam.getOther())); - sqlServerConnectionParam.setUser(sqlServerParam.getUserName()); - sqlServerConnectionParam.setPassword(encodePassword(sqlServerParam.getPassword())); - return sqlServerConnectionParam; - } - - @Override - public BaseConnectionParam createConnectionParams(String connectionJson) { - return JSONUtils.parseObject(connectionJson, SqlServerConnectionParam.class); - } - - @Override - public String getDatasourceDriver() { - return COM_SQLSERVER_JDBC_DRIVER; - } - - @Override - public String getJdbcUrl(ConnectionParam connectionParam) { - SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; - - if (StringUtils.isNotEmpty(sqlServerConnectionParam.getOther())) { - return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther()); - } - return sqlServerConnectionParam.getJdbcUrl(); - } - - @Override - public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { - SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; - Class.forName(getDatasourceDriver()); - return DriverManager.getConnection(getJdbcUrl(connectionParam), sqlServerConnectionParam.getUser(), - decodePassword(sqlServerConnectionParam.getPassword())); - } - - @Override - public DbType getDbType() { - return DbType.SQLSERVER; - } - - private String transformOther(Map otherMap) { - if (MapUtils.isEmpty(otherMap)) { - return null; - } - StringBuilder stringBuilder = new StringBuilder(); - otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); - return stringBuilder.toString(); - } - - private Map parseOther(String other) { - if (StringUtils.isEmpty(other)) { - return null; - } - Map otherMap = new LinkedHashMap<>(); - for (String config : other.split(";")) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - return otherMap; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml index b75f26884c..dceeabc3d1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml @@ -39,6 +39,11 @@ dolphinscheduler-task-api ${project.version}