Browse Source

[feature-6071][plugin] add datasource plugin. (#6071) (#6361)

* [feature-6071][plugin] add datasource plugin. (#6071)

* remove the dependency of datasource in common and task-plugin module.

* the new datasource-api module provides data source connection acquisition.

* add sqlserver data source plug-in.

* add ds license

* fix check style

* update spi pom

* frx conflict resolution

* fix check style

* fix datasource ut

* fix code check style

* fix code check style

* fix datasource ut

* add pg hive mysql oracle clickhouse db2  datasource

* fix datasource package

* update datasource pom

* update datasource pom

* update datasource pom

* update datasource pom

* update pom

* update pom dep

* update pom dep

* fix license

* fix ut

* fix ut

* fix ut

* fix ut

* fix ut

* fix ut

* fix ut

* fix ut

* fix ut

* fix ut

* fix ut

* fix ut

* fix ut

* add oracle package

* modify comments

* conflict resolution

* rename log comment

* rename log comment
3.0.0/version-upgrade
mask 3 years ago committed by GitHub
parent
commit
0aea5ae11a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 6
      dolphinscheduler-api/pom.xml
  2. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
  3. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  4. 13
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java
  5. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java
  6. 48
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
  7. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java
  8. 13
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  9. 98
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseConnectionParam.java
  10. 57
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsConnectionParam.java
  11. 61
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsDatasourceParamDTO.java
  12. 81
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceProcessor.java
  13. 41
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceParamDTO.java
  14. 43
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceParamDTO.java
  15. 38
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveConnectionParam.java
  16. 45
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDataSourceParamDTO.java
  17. 34
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlConnectionParam.java
  18. 43
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceParamDTO.java
  19. 46
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleConnectionParam.java
  20. 55
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceParamDTO.java
  21. 41
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceParamDTO.java
  22. 43
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceParamDTO.java
  23. 38
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkConnectionParam.java
  24. 45
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceParamDTO.java
  25. 43
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceParamDTO.java
  26. 60
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java
  27. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
  28. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
  29. 3
      dolphinscheduler-common/src/main/resources/common.properties
  30. 76
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
  31. 13
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java
  32. 9
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java
  33. 5
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java
  34. 8
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java
  35. 3
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
  36. 9
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java
  37. 8
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java
  38. 2
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java
  39. 177
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml
  40. 129
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java
  41. 12
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDatasourceProcessor.java
  42. 24
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java
  43. 4
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHdfsConnectionParam.java
  44. 2
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHdfsDatasourceParamDTO.java
  45. 14
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DatasourceProcessor.java
  46. 4
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceParamDTO.java
  47. 30
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceProcessor.java
  48. 7
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickhouseConnectionParam.java
  49. 7
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2ConnectionParam.java
  50. 4
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceParamDTO.java
  51. 34
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceProcessor.java
  52. 7
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveConnectionParam.java
  53. 4
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDataSourceParamDTO.java
  54. 40
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDatasourceProcessor.java
  55. 8
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlConnectionParam.java
  56. 4
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceParamDTO.java
  57. 34
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceProcessor.java
  58. 8
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleConnectionParam.java
  59. 4
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceParamDTO.java
  60. 34
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessor.java
  61. 7
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlConnectionParam.java
  62. 4
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceParamDTO.java
  63. 30
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceProcessor.java
  64. 7
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoConnectionParam.java
  65. 4
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceParamDTO.java
  66. 30
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceProcessor.java
  67. 7
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkConnectionParam.java
  68. 4
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceParamDTO.java
  69. 36
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceProcessor.java
  70. 7
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerConnectionParam.java
  71. 4
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceParamDTO.java
  72. 30
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceProcessor.java
  73. 105
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java
  74. 84
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java
  75. 84
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JdbcDataSourceProvider.java
  76. 5
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
  77. 43
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DatasourceUtil.java
  78. 2
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/PasswordUtils.java
  79. 100
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClientTest.java
  80. 26
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java
  81. 25
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceProcessorTest.java
  82. 30
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDatasourceProcessorTest.java
  83. 37
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceProcessorTest.java
  84. 29
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessorTest.java
  85. 27
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java
  86. 27
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceProcessorTest.java
  87. 29
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceProcessorTest.java
  88. 29
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceProcessorTest.java
  89. 52
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JdbcDataSourceProviderTest.java
  90. 124
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtilsTest.java
  91. 42
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DatasourceUtilTest.java
  92. 114
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml
  93. 20
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannel.java
  94. 33
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelFactory.java
  95. 29
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceClient.java
  96. 30
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourcePlugin.java
  97. 33
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelFactoryTest.java
  98. 43
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelTest.java
  99. 37
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourcePluginTest.java
  100. 91
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/pom.xml
  101. Some files were not shown because too many files have changed in this diff Show More

6
dolphinscheduler-api/pom.xml

@ -35,6 +35,12 @@
<artifactId>dolphinscheduler-service</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-api</artifactId>
<version>${project.version}</version>
</dependency>
<!--springboot-->
<dependency>
<groupId>org.springframework.boot</groupId>

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java

@ -34,13 +34,13 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.DataSourceService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.datasource.DatasourceUtil;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.util.Map;

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@ -18,10 +18,10 @@
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.util.Map;

13
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java

@ -22,16 +22,17 @@ import org.apache.dolphinscheduler.api.service.DataSourceService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.datasource.DatasourceUtil;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.lang.StringUtils;
@ -316,7 +317,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
@Override
public Result<Object> checkConnection(DbType type, ConnectionParam connectionParam) {
Result<Object> result = new Result<>();
try (Connection connection = DatasourceUtil.getConnection(type, connectionParam)) {
try (Connection connection = DataSourceClientProvider.getInstance().getConnection(type, connectionParam)) {
if (connection == null) {
putMsg(result, Status.CONNECTION_TEST_FAILURE);
return result;

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java

@ -24,8 +24,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlDatasourceParamDTO;
import java.util.HashMap;

48
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java

@ -21,22 +21,24 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.DataSourceServiceImpl;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.datasource.DatasourceUtil;
import org.apache.dolphinscheduler.common.datasource.hive.HiveDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSqlDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbConnectType;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import java.sql.Connection;
import java.util.ArrayList;
@ -60,7 +62,7 @@ import org.powermock.modules.junit4.PowerMockRunner;
*/
@RunWith(PowerMockRunner.class)
@PowerMockIgnore({"sun.security.*", "javax.net.*"})
@PrepareForTest({DatasourceUtil.class, CommonUtils.class})
@PrepareForTest({DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class DataSourceServiceTest {
@InjectMocks
@ -307,13 +309,14 @@ public class DataSourceServiceTest {
oracleDatasourceParamDTO.setConnectType(DbConnectType.ORACLE_SERVICE_NAME);
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(oracleDatasourceParamDTO);
String expected = "{\"user\":\"test\",\"password\":\"test\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\","
+ "\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"connectType\":\"ORACLE_SERVICE_NAME\"}";
String expected = "{\"user\":\"test\",\"password\":\"test\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\","
+ "\"driverClassName\":\"oracle.jdbc.OracleDriver\",\"validationQuery\":\"select 1 from dual\",\"connectType\":\"ORACLE_SERVICE_NAME\"}";
Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
PowerMockito.mockStatic(CommonUtils.class);
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(true);
PowerMockito.when(CommonUtils.encodePassword(Mockito.anyString())).thenReturn("test");
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO();
hiveDataSourceParamDTO.setHost("192.168.9.1");
hiveDataSourceParamDTO.setPort(10000);
@ -325,8 +328,8 @@ public class DataSourceServiceTest {
hiveDataSourceParamDTO.setLoginUserKeytabPath("/opt/hdfs.headless.keytab");
hiveDataSourceParamDTO.setLoginUserKeytabUsername("test2/hdfs-mycluster@ESZ.COM");
connectionParam = DatasourceUtil.buildConnectionParams(hiveDataSourceParamDTO);
expected = "{\"user\":\"test\",\"password\":\"test\",\"address\":\"jdbc:hive2://192.168.9.1:10000\","
+ "\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\","
expected = "{\"user\":\"test\",\"password\":\"test\",\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;"
+ "principal=hive/hdfs-mycluster@ESZ.COM\",\"driverClassName\":\"org.apache.hive.jdbc.HiveDriver\",\"validationQuery\":\"select 1\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\","
+ "\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\",\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\",\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}";
Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
@ -346,8 +349,8 @@ public class DataSourceServiceTest {
mysqlDatasourceParamDTO.setPassword("123456");
mysqlDatasourceParamDTO.setOther(other);
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO);
String expected = "{\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\",\"address\":\"jdbc:mysql://192.168.9.1:1521\","
+ "\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"}";
String expected = "{\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\",\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/"
+ "im\",\"driverClassName\":\"com.mysql.jdbc.Driver\",\"validationQuery\":\"select 1\",\"props\":{\"autoDeserialize\":\"yes\",\"allowUrlInLocalInfile\":\"true\"}}";
Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
@ -358,8 +361,8 @@ public class DataSourceServiceTest {
mysqlDatasourceParamDTO.setUserName("test");
mysqlDatasourceParamDTO.setPassword("123456");
connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO);
expected = "{\"user\":\"test\",\"password\":\"123456\",\"address\":\"jdbc:mysql://192.168.9.1:1521\","
+ "\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"}";
expected = "{\"user\":\"test\",\"password\":\"123456\",\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\","
+ "\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"driverClassName\":\"com.mysql.jdbc.Driver\",\"validationQuery\":\"select 1\"}";
Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
}
@ -396,12 +399,15 @@ public class DataSourceServiceTest {
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(postgreSqlDatasourceParam);
PowerMockito.mockStatic(DatasourceUtil.class);
PowerMockito.mockStatic(DataSourceClientProvider.class);
DataSourceClientProvider clientProvider = PowerMockito.mock(DataSourceClientProvider.class);
PowerMockito.when(DataSourceClientProvider.getInstance()).thenReturn(clientProvider);
Result result = dataSourceService.checkConnection(dataSourceType, connectionParam);
Assert.assertEquals(Status.CONNECTION_TEST_FAILURE.getCode(), result.getCode().intValue());
Connection connection = PowerMockito.mock(Connection.class);
PowerMockito.when(DatasourceUtil.getConnection(Mockito.any(), Mockito.any())).thenReturn(connection);
PowerMockito.when(clientProvider.getConnection(Mockito.any(), Mockito.any())).thenReturn(connection);
result = dataSourceService.checkConnection(dataSourceType, connectionParam);
Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java

@ -20,10 +20,10 @@ package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.MonitorServiceImpl;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.dao.MonitorDBDao;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.collections.CollectionUtils;

13
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -955,6 +955,19 @@ public final class Constants {
public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver";
public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver";
/**
* validation Query
*/
public static final String POSTGRESQL_VALIDATION_QUERY = "select version()";
public static final String MYSQL_VALIDATION_QUERY = "select 1";
public static final String HIVE_VALIDATION_QUERY = "select 1";
public static final String CLICKHOUSE_VALIDATION_QUERY = "select 1";
public static final String ORACLE_VALIDATION_QUERY = "select 1 from dual";
public static final String SQLSERVER_VALIDATION_QUERY = "select 1";
public static final String DB2_VALIDATION_QUERY = "select 1 from sysibm.sysdummy1";
public static final String PRESTO_VALIDATION_QUERY = "select 1";
/**
* database type
*/

98
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseConnectionParam.java

@ -1,98 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
/**
* The base model of connection param
* <p>
* {@link org.apache.dolphinscheduler.common.datasource.clickhouse.ClickhouseConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.db2.Db2ConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.hive.HiveConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.oracle.OracleConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.presto.PrestoConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.spark.SparkConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerConnectionParam}
*/
@JsonInclude(Include.NON_NULL)
public abstract class BaseConnectionParam implements ConnectionParam {
protected String user;
protected String password;
protected String address;
protected String database;
protected String jdbcUrl;
protected String other;
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getJdbcUrl() {
return jdbcUrl;
}
public void setJdbcUrl(String jdbcUrl) {
this.jdbcUrl = jdbcUrl;
}
public String getOther() {
return other;
}
public void setOther(String other) {
this.other = other;
}
}

57
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsConnectionParam.java

@ -1,57 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
public class BaseHdfsConnectionParam extends BaseConnectionParam {
protected String principal;
protected String javaSecurityKrb5Conf;
protected String loginUserKeytabUsername;
protected String loginUserKeytabPath;
public String getPrincipal() {
return principal;
}
public void setPrincipal(String principal) {
this.principal = principal;
}
public String getJavaSecurityKrb5Conf() {
return javaSecurityKrb5Conf;
}
public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) {
this.javaSecurityKrb5Conf = javaSecurityKrb5Conf;
}
public String getLoginUserKeytabUsername() {
return loginUserKeytabUsername;
}
public void setLoginUserKeytabUsername(String loginUserKeytabUsername) {
this.loginUserKeytabUsername = loginUserKeytabUsername;
}
public String getLoginUserKeytabPath() {
return loginUserKeytabPath;
}
public void setLoginUserKeytabPath(String loginUserKeytabPath) {
this.loginUserKeytabPath = loginUserKeytabPath;
}
}

61
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsDatasourceParamDTO.java

@ -1,61 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
public abstract class BaseHdfsDatasourceParamDTO extends BaseDataSourceParamDTO {
protected String principal;
protected String javaSecurityKrb5Conf;
protected String loginUserKeytabUsername;
protected String loginUserKeytabPath;
public String getPrincipal() {
return principal;
}
public void setPrincipal(String principal) {
this.principal = principal;
}
public String getLoginUserKeytabUsername() {
return loginUserKeytabUsername;
}
public void setLoginUserKeytabUsername(String loginUserKeytabUsername) {
this.loginUserKeytabUsername = loginUserKeytabUsername;
}
public String getLoginUserKeytabPath() {
return loginUserKeytabPath;
}
public void setLoginUserKeytabPath(String loginUserKeytabPath) {
this.loginUserKeytabPath = loginUserKeytabPath;
}
public String getJavaSecurityKrb5Conf() {
return javaSecurityKrb5Conf;
}
public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) {
this.javaSecurityKrb5Conf = javaSecurityKrb5Conf;
}
}

81
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceProcessor.java

@ -1,81 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
public interface DatasourceProcessor {
/**
* check datasource param is valid
*/
void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam);
/**
* create BaseDataSourceParamDTO by connectionJson
*
* @param connectionJson see{@link org.apache.dolphinscheduler.dao.entity.Datasource}
* @return {@link BaseDataSourceParamDTO}
*/
BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson);
/**
* create datasource connection parameter which will be stored at DataSource
* <p>
* see {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams}
*/
ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam);
/**
* deserialize json to datasource connection param
*
* @param connectionJson {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams}
* @return {@link BaseConnectionParam}
*/
ConnectionParam createConnectionParams(String connectionJson);
/**
* get datasource Driver
*/
String getDatasourceDriver();
/**
* get jdbcUrl by connection param, the jdbcUrl is different with ConnectionParam.jdbcUrl, this method will inject
* other to jdbcUrl
*
* @param connectionParam connection param
*/
String getJdbcUrl(ConnectionParam connectionParam);
/**
* get connection by connectionParam
*
* @param connectionParam connectionParam
* @return {@link Connection}
*/
Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException, IOException;
/**
* @return {@link DbType}
*/
DbType getDbType();
}

41
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceParamDTO.java

@ -1,41 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.clickhouse;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class ClickHouseDatasourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "ClickHouseDatasourceParamDTO{"
+ "host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.CLICKHOUSE;
}
}

43
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceParamDTO.java

@ -1,43 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.db2;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class Db2DatasourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "Db2DatasourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.DB2;
}
}

38
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveConnectionParam.java

@ -1,38 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.hive;
import org.apache.dolphinscheduler.common.datasource.BaseHdfsConnectionParam;
public class HiveConnectionParam extends BaseHdfsConnectionParam {
@Override
public String toString() {
return "HiveConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ ", principal='" + principal + '\''
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\''
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\''
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\''
+ '}';
}
}

45
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDataSourceParamDTO.java

@ -1,45 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.hive;
import org.apache.dolphinscheduler.common.datasource.BaseHdfsDatasourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class HiveDataSourceParamDTO extends BaseHdfsDatasourceParamDTO {
@Override
public String toString() {
return "HiveDataSourceParamDTO{"
+ "host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", principal='" + principal + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\''
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\''
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.HIVE;
}
}

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlConnectionParam.java

@ -1,34 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.mysql;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
public class MysqlConnectionParam extends BaseConnectionParam {
@Override
public String toString() {
return "MysqlConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ '}';
}
}

43
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceParamDTO.java

@ -1,43 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.mysql;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class MysqlDatasourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "MysqlDatasourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.MYSQL;
}
}

46
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleConnectionParam.java

@ -1,46 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.oracle;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
public class OracleConnectionParam extends BaseConnectionParam {
protected DbConnectType connectType;
public DbConnectType getConnectType() {
return connectType;
}
public void setConnectType(DbConnectType connectType) {
this.connectType = connectType;
}
@Override
public String toString() {
return "OracleConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ ", connectType=" + connectType
+ '}';
}
}

55
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceParamDTO.java

@ -1,55 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.oracle;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
public class OracleDatasourceParamDTO extends BaseDataSourceParamDTO {
private DbConnectType connectType;
public DbConnectType getConnectType() {
return connectType;
}
public void setConnectType(DbConnectType connectType) {
this.connectType = connectType;
}
@Override
public String toString() {
return "OracleDatasourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", connectType=" + connectType
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.ORACLE;
}
}

41
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceParamDTO.java

@ -1,41 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.postgresql;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class PostgreSqlDatasourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "PostgreSqlDatasourceParamDTO{"
+ "host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.POSTGRESQL;
}
}

43
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceParamDTO.java

@ -1,43 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.presto;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class PrestoDatasourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "PrestoDatasourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.PRESTO;
}
}

38
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkConnectionParam.java

@ -1,38 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.spark;
import org.apache.dolphinscheduler.common.datasource.BaseHdfsConnectionParam;
public class SparkConnectionParam extends BaseHdfsConnectionParam {
@Override
public String toString() {
return "SparkConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ ", principal='" + principal + '\''
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\''
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\''
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\''
+ '}';
}
}

45
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceParamDTO.java

@ -1,45 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.spark;
import org.apache.dolphinscheduler.common.datasource.BaseHdfsDatasourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class SparkDatasourceParamDTO extends BaseHdfsDatasourceParamDTO {
@Override
public String toString() {
return "SparkDatasourceParamDTO{"
+ "host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", principal='" + principal + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\''
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\''
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.SPARK;
}
}

43
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceParamDTO.java

@ -1,43 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.sqlserver;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class SqlServerDatasourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "SqlServerDatasourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.SQLSERVER;
}
}

60
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.enums;
import static java.util.stream.Collectors.toMap;
import java.util.Arrays;
import java.util.Map;
import com.baomidou.mybatisplus.annotation.EnumValue;
import com.google.common.base.Functions;
public enum DbType {
MYSQL(0),
POSTGRESQL(1),
HIVE(2),
SPARK(3),
CLICKHOUSE(4),
ORACLE(5),
SQLSERVER(6),
DB2(7),
PRESTO(8),
H2(9);
DbType(int code) {
this.code = code;
}
@EnumValue
private final int code;
public int getCode() {
return code;
}
private static final Map<Integer, DbType> DB_TYPE_MAP =
Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity()));
public static DbType of(int type) {
if (DB_TYPE_MAP.containsKey(type)) {
return DB_TYPE_MAP.get(type);
}
return null;
}
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java

@ -18,7 +18,7 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.spi.enums.ResUploadType;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.spi.enums.ResUploadType;
import org.apache.directory.api.util.Strings;

3
dolphinscheduler-common/src/main/resources/common.properties

@ -89,3 +89,6 @@ sudo.enable=true
# development state
development.state=false
#datasource.plugin.dir config
datasource.plugin.dir=lib/plugin/datasource

76
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java

@ -17,9 +17,8 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import java.net.InetAddress;
@ -28,7 +27,6 @@ import java.net.UnknownHostException;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
@ -53,41 +51,6 @@ public class CommonUtilsTest {
Assert.assertTrue(true);
}
@Test
public void getKerberosStartupState() {
boolean kerberosStartupState = CommonUtils.getKerberosStartupState();
logger.info("kerberos startup state: {}",kerberosStartupState);
Assert.assertFalse(kerberosStartupState);
PowerMockito.mockStatic(PropertyUtils.class);
PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS");
PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE);
kerberosStartupState = CommonUtils.getKerberosStartupState();
logger.info("kerberos startup state: {}",kerberosStartupState);
Assert.assertTrue(kerberosStartupState);
}
@Test
public void loadKerberosConf() {
try {
PowerMockito.mockStatic(PropertyUtils.class);
PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS");
PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE);
PowerMockito.when(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)).thenReturn("/opt/krb5.conf");
PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)).thenReturn("hdfs-mycluster@ESZ.COM");
PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)).thenReturn("/opt/hdfs.headless.keytab");
PowerMockito.mockStatic(UserGroupInformation.class);
boolean result = CommonUtils.loadKerberosConf(new Configuration());
Assert.assertTrue(result);
CommonUtils.loadKerberosConf(null, null, null);
} catch (Exception e) {
Assert.fail("load Kerberos Conf failed");
}
}
@Test
public void getHdfsDataBasePath() {
logger.info(HadoopUtils.getHdfsDataBasePath());
@ -124,41 +87,4 @@ public class CommonUtilsTest {
Assert.assertTrue(true);
}
@Test
public void encodePassword() {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
Assert.assertEquals("",CommonUtils.encodePassword(""));
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==",CommonUtils.encodePassword("123456"));
Assert.assertEquals("IUAjJCVeJipJVkZCV2xoVFYwQT0=",CommonUtils.encodePassword("!QAZXSW@"));
Assert.assertEquals("IUAjJCVeJipOV1JtWjJWeUtFQT0=",CommonUtils.encodePassword("5dfger(@"));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
Assert.assertEquals("",CommonUtils.encodePassword(""));
Assert.assertEquals("123456",CommonUtils.encodePassword("123456"));
Assert.assertEquals("!QAZXSW@",CommonUtils.encodePassword("!QAZXSW@"));
Assert.assertEquals("5dfger(@",CommonUtils.encodePassword("5dfger(@"));
}
@Test
public void decodePassword() {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
Assert.assertEquals("", CommonUtils.decodePassword(""));
Assert.assertEquals("123456", CommonUtils.decodePassword("IUAjJCVeJipNVEl6TkRVMg=="));
Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("IUAjJCVeJipJVkZCV2xoVFYwQT0="));
Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("IUAjJCVeJipOV1JtWjJWeUtFQT0="));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
Assert.assertEquals("", CommonUtils.decodePassword(""));
Assert.assertEquals("123456", CommonUtils.decodePassword("123456"));
Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("!QAZXSW@"));
Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("5dfger(@"));
}
}

13
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java

@ -16,20 +16,23 @@
*/
package org.apache.dolphinscheduler.dao;
import com.alibaba.druid.pool.DruidDataSource;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.List;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.apache.dolphinscheduler.dao.utils.MysqlPerformance;
import org.apache.dolphinscheduler.dao.utils.PostgrePerformance;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.alibaba.druid.pool.DruidDataSource;
/**
* database state dao

9
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java

@ -16,14 +16,15 @@
*/
package org.apache.dolphinscheduler.dao.entity;
import com.fasterxml.jackson.annotation.JsonFormat;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.util.Date;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.util.Date;
import com.fasterxml.jackson.annotation.JsonFormat;
@TableName("t_ds_datasource")
public class DataSource {

5
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java

@ -16,12 +16,13 @@
*/
package org.apache.dolphinscheduler.dao.entity;
import com.fasterxml.jackson.annotation.JsonFormat;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.util.Date;
import com.fasterxml.jackson.annotation.JsonFormat;
/**
* monitor record for database
*/

8
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java

@ -14,15 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.SchemaUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* upgrade manager
*/

3
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@ -14,11 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ConditionType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.TaskType;
@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;

9
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java

@ -14,20 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.utils;
package org.apache.dolphinscheduler.dao.utils;
import static org.apache.dolphinscheduler.dao.MonitorDBDao.VARIABLE_NAME;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

8
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java

@ -14,17 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.utils;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

2
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java

@ -24,13 +24,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.BaseDaoTest;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.DatasourceUser;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.util.Arrays;
import java.util.Date;

177
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml

@ -0,0 +1,177 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dolphinscheduler-datasource-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dolphinscheduler-datasource-api</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.github.oshi</groupId>
<artifactId>oshi-core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
</exclusion>
<exclusion>
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
</exclusion>
<exclusion>
<artifactId>jersey-client</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jaxb-api</artifactId>
<groupId>javax.xml.bind</groupId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<type>jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jacoco</groupId>
<artifactId>org.jacoco.agent</artifactId>
<classifier>runtime</classifier>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>dolphinscheduler-datasource-api-${project.version}</finalName>
</build>
</project>

129
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java

@ -0,0 +1,129 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.client;
import org.apache.dolphinscheduler.plugin.datasource.api.provider.JdbcDataSourceProvider;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.DataSourceClient;
import org.apache.dolphinscheduler.spi.exception.PluginException;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.JdbcTemplate;
import com.alibaba.druid.pool.DruidDataSource;
import com.google.common.base.Stopwatch;
public class CommonDataSourceClient implements DataSourceClient {
private static final Logger logger = LoggerFactory.getLogger(CommonDataSourceClient.class);
public static final String COMMON_USER = "root";
public static final String COMMON_PASSWORD = "123456";
public static final String COMMON_VALIDATION_QUERY = "select 1";
protected final BaseConnectionParam baseConnectionParam;
protected DruidDataSource druidDataSource;
protected JdbcTemplate jdbcTemplate;
public CommonDataSourceClient(BaseConnectionParam baseConnectionParam) {
this.baseConnectionParam = baseConnectionParam;
preInit();
checkEnv(baseConnectionParam);
initClient(baseConnectionParam);
checkClient();
}
protected void preInit() {
logger.info("preInit in CommonDataSourceClient");
}
protected void checkEnv(BaseConnectionParam baseConnectionParam) {
checkValidationQuery(baseConnectionParam);
checkUser(baseConnectionParam);
}
protected void initClient(BaseConnectionParam baseConnectionParam) {
this.druidDataSource = JdbcDataSourceProvider.createJdbcDataSource(baseConnectionParam);
this.jdbcTemplate = new JdbcTemplate(druidDataSource);
}
protected void checkUser(BaseConnectionParam baseConnectionParam) {
if (StringUtils.isBlank(baseConnectionParam.getUser())) {
setDefaultUsername(baseConnectionParam);
}
if (StringUtils.isBlank(baseConnectionParam.getPassword())) {
setDefaultPassword(baseConnectionParam);
}
}
protected void setDefaultUsername(BaseConnectionParam baseConnectionParam) {
baseConnectionParam.setUser(COMMON_USER);
}
protected void setDefaultPassword(BaseConnectionParam baseConnectionParam) {
baseConnectionParam.setPassword(COMMON_PASSWORD);
}
protected void checkValidationQuery(BaseConnectionParam baseConnectionParam) {
if (StringUtils.isBlank(baseConnectionParam.getValidationQuery())) {
setDefaultValidationQuery(baseConnectionParam);
}
}
protected void setDefaultValidationQuery(BaseConnectionParam baseConnectionParam) {
baseConnectionParam.setValidationQuery(COMMON_VALIDATION_QUERY);
}
@Override
public void checkClient() {
//Checking data source client
Stopwatch stopwatch = Stopwatch.createStarted();
try {
this.jdbcTemplate.execute(this.baseConnectionParam.getValidationQuery());
} catch (Exception e) {
throw PluginException.getInstance("JDBC connect failed", e);
} finally {
logger.info("Time to execute check jdbc client with sql {} for {} ms ", this.baseConnectionParam.getValidationQuery(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
}
}
@Override
public Connection getConnection() {
try {
return this.druidDataSource.getConnection();
} catch (SQLException e) {
logger.error("get druidDataSource Connection fail SQLException: {}", e.getMessage(), e);
return null;
}
}
@Override
public void close() {
logger.info("do close dataSource.");
this.druidDataSource.close();
this.druidDataSource = null;
this.jdbcTemplate = null;
}
}

12
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDatasourceProcessor.java

@ -15,10 +15,15 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.collections4.MapUtils;
import java.text.MessageFormat;
import java.util.Map;
import java.util.regex.Pattern;
@ -76,4 +81,9 @@ public abstract class AbstractDatasourceProcessor implements DatasourceProcessor
}
}
@Override
public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) {
BaseConnectionParam baseConnectionParam = (BaseConnectionParam) connectionParam;
return MessageFormat.format("{0}@{1}@{2}", dbType.getDescp(), baseConnectionParam.getUser(), baseConnectionParam.getJdbcUrl());
}
}

24
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java

@ -15,18 +15,18 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
import org.apache.dolphinscheduler.common.datasource.clickhouse.ClickHouseDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.db2.Db2DatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.hive.HiveDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.presto.PrestoDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.spark.SparkDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerDatasourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickHouseDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2DatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSqlDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SqlServerDatasourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.io.Serializable;
import java.util.Map;

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseHdfsConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHdfsConnectionParam.java

@ -15,7 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
public class BaseHdfsConnectionParam extends BaseConnectionParam {
protected String principal;

2
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/BaseHdfsDatasourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHdfsDatasourceParamDTO.java

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
public abstract class BaseHdfsDatasourceParamDTO extends BaseDataSourceParamDTO {

14
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/DatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DatasourceProcessor.java

@ -15,8 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.io.IOException;
@ -30,6 +31,12 @@ public interface DatasourceProcessor {
*/
void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam);
/**
* get Datasource Client UniqueId
* @return UniqueId
*/
String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType);
/**
* create BaseDataSourceParamDTO by connectionJson
*
@ -58,6 +65,11 @@ public interface DatasourceProcessor {
*/
String getDatasourceDriver();
/**
* get validation Query
*/
String getValidationQuery();
/**
* get jdbcUrl by connection param, the jdbcUrl is different with ConnectionParam.jdbcUrl, this method will inject
* other to jdbcUrl

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickHouseDatasourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceParamDTO.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.clickhouse;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class ClickHouseDatasourceParamDTO extends BaseDataSourceParamDTO {

30
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceProcessor.java

@ -15,18 +15,18 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.clickhouse;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -64,8 +64,11 @@ public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor {
clickhouseConnectionParam.setAddress(address);
clickhouseConnectionParam.setJdbcUrl(jdbcUrl);
clickhouseConnectionParam.setUser(clickHouseParam.getUserName());
clickhouseConnectionParam.setPassword(CommonUtils.encodePassword(clickHouseParam.getPassword()));
clickhouseConnectionParam.setPassword(PasswordUtils.encodePassword(clickHouseParam.getPassword()));
clickhouseConnectionParam.setDriverClassName(getDatasourceDriver());
clickhouseConnectionParam.setValidationQuery(getValidationQuery());
clickhouseConnectionParam.setOther(transformOther(clickHouseParam.getOther()));
clickhouseConnectionParam.setProps(clickHouseParam.getOther());
return clickhouseConnectionParam;
}
@ -79,6 +82,11 @@ public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor {
return Constants.COM_CLICKHOUSE_JDBC_DRIVER;
}
@Override
public String getValidationQuery() {
return Constants.CLICKHOUSE_VALIDATION_QUERY;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam;
@ -94,7 +102,7 @@ public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor {
ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam),
clickhouseConnectionParam.getUser(), CommonUtils.decodePassword(clickhouseConnectionParam.getPassword()));
clickhouseConnectionParam.getUser(), PasswordUtils.decodePassword(clickhouseConnectionParam.getPassword()));
}
@Override

7
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/clickhouse/ClickhouseConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickhouseConnectionParam.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.clickhouse;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
public class ClickhouseConnectionParam extends BaseConnectionParam {
@Override
@ -28,6 +28,9 @@ public class ClickhouseConnectionParam extends BaseConnectionParam {
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ '}';
}

7
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2ConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2ConnectionParam.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.db2;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
public class Db2ConnectionParam extends BaseConnectionParam {
@Override
@ -28,6 +28,9 @@ public class Db2ConnectionParam extends BaseConnectionParam {
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ '}';
}

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/db2/Db2DatasourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceParamDTO.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.db2;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class Db2DatasourceParamDTO extends BaseDataSourceParamDTO {

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceProcessor.java

@ -15,19 +15,19 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.db2;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -65,8 +65,11 @@ public class Db2DatasourceProcessor extends AbstractDatasourceProcessor {
db2ConnectionParam.setDatabase(db2Param.getDatabase());
db2ConnectionParam.setJdbcUrl(jdbcUrl);
db2ConnectionParam.setUser(db2Param.getUserName());
db2ConnectionParam.setPassword(CommonUtils.encodePassword(db2Param.getPassword()));
db2ConnectionParam.setPassword(PasswordUtils.encodePassword(db2Param.getPassword()));
db2ConnectionParam.setDriverClassName(getDatasourceDriver());
db2ConnectionParam.setValidationQuery(getValidationQuery());
db2ConnectionParam.setOther(transformOther(db2Param.getOther()));
db2ConnectionParam.setProps(db2Param.getOther());
return db2ConnectionParam;
}
@ -95,7 +98,7 @@ public class Db2DatasourceProcessor extends AbstractDatasourceProcessor {
Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(db2ConnectionParam),
db2ConnectionParam.getUser(), CommonUtils.decodePassword(db2ConnectionParam.getPassword()));
db2ConnectionParam.getUser(), PasswordUtils.decodePassword(db2ConnectionParam.getPassword()));
}
@Override
@ -103,6 +106,11 @@ public class Db2DatasourceProcessor extends AbstractDatasourceProcessor {
return DbType.DB2;
}
@Override
public String getValidationQuery() {
return Constants.DB2_VALIDATION_QUERY;
}
private String transformOther(Map<String, String> otherMap) {
if (MapUtils.isEmpty(otherMap)) {
return null;

7
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveConnectionParam.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.hive;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseHdfsConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHdfsConnectionParam;
public class HiveConnectionParam extends BaseHdfsConnectionParam {
@Override
@ -28,6 +28,9 @@ public class HiveConnectionParam extends BaseHdfsConnectionParam {
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ ", principal='" + principal + '\''
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\''

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/hive/HiveDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDataSourceParamDTO.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.hive;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseHdfsDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHdfsDatasourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class HiveDataSourceParamDTO extends BaseHdfsDatasourceParamDTO {

40
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDatasourceProcessor.java

@ -15,20 +15,20 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.hive;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.HiveConfUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.sql.Connection;
@ -83,7 +83,9 @@ public class HiveDatasourceProcessor extends AbstractDatasourceProcessor {
hiveConnectionParam.setAddress(address.toString());
hiveConnectionParam.setJdbcUrl(jdbcUrl);
hiveConnectionParam.setUser(hiveParam.getUserName());
hiveConnectionParam.setPassword(CommonUtils.encodePassword(hiveParam.getPassword()));
hiveConnectionParam.setPassword(PasswordUtils.encodePassword(hiveParam.getPassword()));
hiveConnectionParam.setDriverClassName(getDatasourceDriver());
hiveConnectionParam.setValidationQuery(getValidationQuery());
if (CommonUtils.getKerberosStartupState()) {
hiveConnectionParam.setPrincipal(hiveParam.getPrincipal());
@ -92,6 +94,7 @@ public class HiveDatasourceProcessor extends AbstractDatasourceProcessor {
hiveConnectionParam.setLoginUserKeytabUsername(hiveParam.getLoginUserKeytabUsername());
}
hiveConnectionParam.setOther(transformOther(hiveParam.getOther()));
hiveConnectionParam.setProps(hiveParam.getOther());
return hiveConnectionParam;
}
@ -105,6 +108,11 @@ public class HiveDatasourceProcessor extends AbstractDatasourceProcessor {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
}
@Override
public String getValidationQuery() {
return Constants.HIVE_VALIDATION_QUERY;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam;
@ -123,7 +131,7 @@ public class HiveDatasourceProcessor extends AbstractDatasourceProcessor {
hiveConnectionParam.getLoginUserKeytabUsername(), hiveConnectionParam.getLoginUserKeytabPath());
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(connectionParam),
hiveConnectionParam.getUser(), CommonUtils.decodePassword(hiveConnectionParam.getPassword()));
hiveConnectionParam.getUser(), PasswordUtils.decodePassword(hiveConnectionParam.getPassword()));
}
@Override
@ -152,12 +160,8 @@ public class HiveDatasourceProcessor extends AbstractDatasourceProcessor {
String[] otherArray = otherParams.split(";", -1);
for (String conf : otherArray) {
if (HiveConfUtils.isHiveConfVar(conf)) {
hiveConfListSb.append(conf).append(";");
} else {
sessionVarListSb.append(conf).append(";");
}
}
// remove the last ";"
if (sessionVarListSb.length() > 0) {

8
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlConnectionParam.java

@ -15,12 +15,11 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.mysql;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
public class MysqlConnectionParam extends BaseConnectionParam {
@Override
public String toString() {
return "MysqlConnectionParam{"
@ -29,6 +28,9 @@ public class MysqlConnectionParam extends BaseConnectionParam {
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ '}';
}

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/mysql/MysqlDatasourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceParamDTO.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.mysql;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class MysqlDatasourceParamDTO extends BaseDataSourceParamDTO {

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceProcessor.java

@ -15,19 +15,19 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.mysql;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -82,8 +82,11 @@ public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor {
mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase());
mysqlConnectionParam.setAddress(address);
mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName());
mysqlConnectionParam.setPassword(CommonUtils.encodePassword(mysqlDatasourceParam.getPassword()));
mysqlConnectionParam.setPassword(PasswordUtils.encodePassword(mysqlDatasourceParam.getPassword()));
mysqlConnectionParam.setDriverClassName(getDatasourceDriver());
mysqlConnectionParam.setValidationQuery(getValidationQuery());
mysqlConnectionParam.setOther(transformOther(mysqlDatasourceParam.getOther()));
mysqlConnectionParam.setProps(mysqlDatasourceParam.getOther());
return mysqlConnectionParam;
}
@ -98,6 +101,11 @@ public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor {
return Constants.COM_MYSQL_JDBC_DRIVER;
}
@Override
public String getValidationQuery() {
return Constants.MYSQL_VALIDATION_QUERY;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam;
@ -117,7 +125,7 @@ public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor {
logger.warn("sensitive param : {} in username field is filtered", AUTO_DESERIALIZE);
user = user.replace(AUTO_DESERIALIZE, "");
}
String password = CommonUtils.decodePassword(mysqlConnectionParam.getPassword());
String password = PasswordUtils.decodePassword(mysqlConnectionParam.getPassword());
if (password.contains(AUTO_DESERIALIZE)) {
logger.warn("sensitive param : {} in password field is filtered", AUTO_DESERIALIZE);
password = password.replace(AUTO_DESERIALIZE, "");

8
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleConnectionParam.java

@ -15,13 +15,12 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.oracle;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbConnectType;
public class OracleConnectionParam extends BaseConnectionParam {
protected DbConnectType connectType;
public DbConnectType getConnectType() {
@ -40,6 +39,9 @@ public class OracleConnectionParam extends BaseConnectionParam {
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ ", connectType=" + connectType
+ '}';

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/oracle/OracleDatasourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceParamDTO.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.oracle;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbConnectType;
import org.apache.dolphinscheduler.spi.enums.DbType;

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessor.java

@ -15,17 +15,17 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.oracle;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbConnectType;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
@ -76,12 +76,15 @@ public class OracleDatasourceProcessor extends AbstractDatasourceProcessor {
OracleConnectionParam oracleConnectionParam = new OracleConnectionParam();
oracleConnectionParam.setUser(oracleParam.getUserName());
oracleConnectionParam.setPassword(CommonUtils.encodePassword(oracleParam.getPassword()));
oracleConnectionParam.setPassword(PasswordUtils.encodePassword(oracleParam.getPassword()));
oracleConnectionParam.setAddress(address);
oracleConnectionParam.setJdbcUrl(jdbcUrl);
oracleConnectionParam.setDatabase(oracleParam.getDatabase());
oracleConnectionParam.setConnectType(oracleParam.getConnectType());
oracleConnectionParam.setDriverClassName(getDatasourceDriver());
oracleConnectionParam.setValidationQuery(getValidationQuery());
oracleConnectionParam.setOther(transformOther(oracleParam.getOther()));
oracleConnectionParam.setProps(oracleParam.getOther());
return oracleConnectionParam;
}
@ -96,6 +99,11 @@ public class OracleDatasourceProcessor extends AbstractDatasourceProcessor {
return Constants.COM_ORACLE_JDBC_DRIVER;
}
@Override
public String getValidationQuery() {
return Constants.ORACLE_VALIDATION_QUERY;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam;
@ -110,7 +118,7 @@ public class OracleDatasourceProcessor extends AbstractDatasourceProcessor {
OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(connectionParam),
oracleConnectionParam.getUser(), CommonUtils.decodePassword(oracleConnectionParam.getPassword()));
oracleConnectionParam.getUser(), PasswordUtils.decodePassword(oracleConnectionParam.getPassword()));
}
@Override

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlConnectionParam.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.postgresql;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
public class PostgreSqlConnectionParam extends BaseConnectionParam {
@Override
@ -28,6 +28,9 @@ public class PostgreSqlConnectionParam extends BaseConnectionParam {
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ '}';
}

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/postgresql/PostgreSqlDatasourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceParamDTO.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.postgresql;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class PostgreSqlDatasourceParamDTO extends BaseDataSourceParamDTO {

30
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceProcessor.java

@ -15,16 +15,16 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.postgresql;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
@ -65,8 +65,11 @@ public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor {
postgreSqlConnectionParam.setAddress(address);
postgreSqlConnectionParam.setDatabase(postgreSqlParam.getDatabase());
postgreSqlConnectionParam.setUser(postgreSqlParam.getUserName());
postgreSqlConnectionParam.setPassword(CommonUtils.encodePassword(postgreSqlParam.getPassword()));
postgreSqlConnectionParam.setPassword(PasswordUtils.encodePassword(postgreSqlParam.getPassword()));
postgreSqlConnectionParam.setDriverClassName(getDatasourceDriver());
postgreSqlConnectionParam.setValidationQuery(getValidationQuery());
postgreSqlConnectionParam.setOther(transformOther(postgreSqlParam.getOther()));
postgreSqlConnectionParam.setProps(postgreSqlParam.getOther());
return postgreSqlConnectionParam;
}
@ -81,6 +84,11 @@ public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor {
return Constants.ORG_POSTGRESQL_DRIVER;
}
@Override
public String getValidationQuery() {
return Constants.POSTGRESQL_VALIDATION_QUERY;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam;
@ -95,7 +103,7 @@ public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor {
PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam),
postgreSqlConnectionParam.getUser(), CommonUtils.decodePassword(postgreSqlConnectionParam.getPassword()));
postgreSqlConnectionParam.getUser(), PasswordUtils.decodePassword(postgreSqlConnectionParam.getPassword()));
}
@Override

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoConnectionParam.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.presto;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
public class PrestoConnectionParam extends BaseConnectionParam {
@Override
@ -28,6 +28,9 @@ public class PrestoConnectionParam extends BaseConnectionParam {
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ '}';
}

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/presto/PrestoDatasourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceParamDTO.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.presto;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class PrestoDatasourceParamDTO extends BaseDataSourceParamDTO {

30
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceProcessor.java

@ -15,16 +15,16 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.presto;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
@ -64,11 +64,14 @@ public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor {
PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam();
prestoConnectionParam.setUser(prestoParam.getUserName());
prestoConnectionParam.setPassword(CommonUtils.encodePassword(prestoParam.getPassword()));
prestoConnectionParam.setPassword(PasswordUtils.encodePassword(prestoParam.getPassword()));
prestoConnectionParam.setOther(transformOther(prestoParam.getOther()));
prestoConnectionParam.setAddress(address);
prestoConnectionParam.setJdbcUrl(jdbcUrl);
prestoConnectionParam.setDatabase(prestoParam.getDatabase());
prestoConnectionParam.setDriverClassName(getDatasourceDriver());
prestoConnectionParam.setValidationQuery(getValidationQuery());
prestoConnectionParam.setProps(prestoParam.getOther());
return prestoConnectionParam;
}
@ -83,6 +86,11 @@ public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor {
return Constants.COM_PRESTO_JDBC_DRIVER;
}
@Override
public String getValidationQuery() {
return Constants.PRESTO_VALIDATION_QUERY;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam;
@ -97,7 +105,7 @@ public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor {
PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(connectionParam),
prestoConnectionParam.getUser(), CommonUtils.decodePassword(prestoConnectionParam.getPassword()));
prestoConnectionParam.getUser(), PasswordUtils.decodePassword(prestoConnectionParam.getPassword()));
}
@Override

7
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkConnectionParam.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.spark;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseHdfsConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHdfsConnectionParam;
public class SparkConnectionParam extends BaseHdfsConnectionParam {
@Override
@ -28,6 +28,9 @@ public class SparkConnectionParam extends BaseHdfsConnectionParam {
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ ", principal='" + principal + '\''
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\''

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/spark/SparkDatasourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceParamDTO.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.spark;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseHdfsDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHdfsDatasourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class SparkDatasourceParamDTO extends BaseHdfsDatasourceParamDTO {

36
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceProcessor.java

@ -15,19 +15,20 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.spark;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.sql.Connection;
@ -81,12 +82,16 @@ public class SparkDatasourceProcessor extends AbstractDatasourceProcessor {
}
SparkConnectionParam sparkConnectionParam = new SparkConnectionParam();
sparkConnectionParam.setPassword(CommonUtils.encodePassword(sparkDatasourceParam.getPassword()));
sparkConnectionParam.setPassword(PasswordUtils.encodePassword(sparkDatasourceParam.getPassword()));
sparkConnectionParam.setUser(sparkDatasourceParam.getUserName());
sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther()));
sparkConnectionParam.setDatabase(sparkDatasourceParam.getDatabase());
sparkConnectionParam.setAddress(address.toString());
sparkConnectionParam.setJdbcUrl(jdbcUrl);
sparkConnectionParam.setDriverClassName(getDatasourceDriver());
sparkConnectionParam.setValidationQuery(getValidationQuery());
sparkConnectionParam.setProps(sparkDatasourceParam.getOther());
if (CommonUtils.getKerberosStartupState()) {
sparkConnectionParam.setPrincipal(sparkDatasourceParam.getPrincipal());
sparkConnectionParam.setJavaSecurityKrb5Conf(sparkDatasourceParam.getJavaSecurityKrb5Conf());
@ -107,6 +112,11 @@ public class SparkDatasourceProcessor extends AbstractDatasourceProcessor {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
}
@Override
public String getValidationQuery() {
return Constants.HIVE_VALIDATION_QUERY;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam;
@ -123,7 +133,7 @@ public class SparkDatasourceProcessor extends AbstractDatasourceProcessor {
sparkConnectionParam.getLoginUserKeytabUsername(), sparkConnectionParam.getLoginUserKeytabPath());
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(sparkConnectionParam),
sparkConnectionParam.getUser(), CommonUtils.decodePassword(sparkConnectionParam.getPassword()));
sparkConnectionParam.getUser(), PasswordUtils.decodePassword(sparkConnectionParam.getPassword()));
}
@Override

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerConnectionParam.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.sqlserver;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
public class SqlServerConnectionParam extends BaseConnectionParam {
@Override
@ -28,6 +28,9 @@ public class SqlServerConnectionParam extends BaseConnectionParam {
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ '}';
}

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/sqlserver/SqlServerDatasourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceParamDTO.java

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource.sqlserver;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver;
import org.apache.dolphinscheduler.plugin.task.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
public class SqlServerDatasourceParamDTO extends BaseDataSourceParamDTO {

30
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceProcessor.java

@ -15,16 +15,16 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.sqlserver;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
@ -64,7 +64,10 @@ public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor {
sqlServerConnectionParam.setJdbcUrl(jdbcUrl);
sqlServerConnectionParam.setOther(transformOther(sqlServerParam.getOther()));
sqlServerConnectionParam.setUser(sqlServerParam.getUserName());
sqlServerConnectionParam.setPassword(CommonUtils.encodePassword(sqlServerParam.getPassword()));
sqlServerConnectionParam.setPassword(PasswordUtils.encodePassword(sqlServerParam.getPassword()));
sqlServerConnectionParam.setDriverClassName(getDatasourceDriver());
sqlServerConnectionParam.setValidationQuery(getValidationQuery());
sqlServerConnectionParam.setProps(sqlServerParam.getOther());
return sqlServerConnectionParam;
}
@ -78,6 +81,11 @@ public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor {
return Constants.COM_SQLSERVER_JDBC_DRIVER;
}
@Override
public String getValidationQuery() {
return Constants.SQLSERVER_VALIDATION_QUERY;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam;
@ -93,7 +101,7 @@ public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor {
SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(connectionParam), sqlServerConnectionParam.getUser(),
CommonUtils.decodePassword(sqlServerConnectionParam.getPassword()));
PasswordUtils.decodePassword(sqlServerConnectionParam.getPassword()));
}
@Override

105
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java

@ -0,0 +1,105 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.plugin;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
import org.apache.dolphinscheduler.spi.datasource.DataSourceClient;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.exception.PluginException;
import org.apache.dolphinscheduler.spi.plugin.DolphinPluginLoader;
import org.apache.dolphinscheduler.spi.plugin.DolphinPluginManagerConfig;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableList;
public class DataSourceClientProvider {
private static final Logger logger = LoggerFactory.getLogger(DataSourceClientProvider.class);
private static final Map<String, DataSourceClient> uniqueId2dataSourceClientMap = new ConcurrentHashMap<>();
private DataSourcePluginManager dataSourcePluginManager;
private DataSourceClientProvider() {
initDataSourcePlugin();
}
private static class DataSourceClientProviderHolder {
private static final DataSourceClientProvider INSTANCE = new DataSourceClientProvider();
}
public static DataSourceClientProvider getInstance() {
return DataSourceClientProviderHolder.INSTANCE;
}
public Connection getConnection(DbType dbType, ConnectionParam connectionParam) {
BaseConnectionParam baseConnectionParam = (BaseConnectionParam) connectionParam;
String datasourceUniqueId = DatasourceUtil.getDatasourceUniqueId(baseConnectionParam, dbType);
logger.info("getConnection datasourceUniqueId {}", datasourceUniqueId);
DataSourceClient dataSourceClient;
dataSourceClient = uniqueId2dataSourceClientMap.get(datasourceUniqueId);
if (dataSourceClient == null) {
Map<String, DataSourceChannel> dataSourceChannelMap = dataSourcePluginManager.getDataSourceChannelMap();
DataSourceChannel dataSourceChannel = dataSourceChannelMap.get(dbType.getDescp());
if (null == dataSourceChannel) {
throw PluginException.getInstance(String.format("%s dataSource Plugin Not Found,Please Check Config File.", dbType.getDescp()));
}
dataSourceClient = dataSourceChannel.createDataSourceClient(baseConnectionParam);
uniqueId2dataSourceClientMap.put(datasourceUniqueId, dataSourceClient);
}
return dataSourceClient.getConnection();
}
private void initDataSourcePlugin() {
dataSourcePluginManager = new DataSourcePluginManager();
DolphinPluginManagerConfig datasourcePluginManagerConfig = new DolphinPluginManagerConfig();
datasourcePluginManagerConfig.setPlugins(PropertyUtils.getString(Constants.DATASOURCE_PLUGIN_BINDING));
datasourcePluginManagerConfig.setInstalledPluginsDir(PropertyUtils.getString(Constants.DATASOURCE_PLUGIN_DIR, Constants.DATASOURCE_PLUGIN_PATH));
if (StringUtils.isNotBlank(PropertyUtils.getString(Constants.MAVEN_LOCAL_REPOSITORY))) {
datasourcePluginManagerConfig.setMavenLocalRepository(PropertyUtils.getString(Constants.MAVEN_LOCAL_REPOSITORY).trim());
}
DolphinPluginLoader datasourcePluginLoader = new DolphinPluginLoader(datasourcePluginManagerConfig, ImmutableList.of(dataSourcePluginManager));
try {
datasourcePluginLoader.loadPlugins();
} catch (Exception e) {
throw PluginException.getInstance("Load datasource Plugin Failed !", e);
}
if (MapUtils.isEmpty(dataSourcePluginManager.getDataSourceChannelMap())) {
throw PluginException.getInstance("datasource Plugin Not Found,Please Check Config File");
}
}
}

84
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java

@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.plugin;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
import static com.google.common.base.Preconditions.checkState;
import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin;
import org.apache.dolphinscheduler.spi.classloader.ThreadContextClassLoader;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
import org.apache.dolphinscheduler.spi.exception.PluginException;
import org.apache.dolphinscheduler.spi.plugin.AbstractDolphinPluginManager;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataSourcePluginManager extends AbstractDolphinPluginManager {
private static final Logger logger = LoggerFactory.getLogger(DataSourcePluginManager.class);
private final Map<String, DataSourceChannelFactory> datasourceClientFactoryMap = new ConcurrentHashMap<>();
private final Map<String, DataSourceChannel> datasourceClientMap = new ConcurrentHashMap<>();
public Map<String, DataSourceChannel> getDataSourceChannelMap() {
return datasourceClientMap;
}
private void addDatasourceClientFactory(DataSourceChannelFactory datasourceChannelFactory) {
requireNonNull(datasourceChannelFactory, "datasourceChannelFactory is null");
if (datasourceClientFactoryMap.putIfAbsent(datasourceChannelFactory.getName(), datasourceChannelFactory) != null) {
throw PluginException.getInstance(format("Datasource Plugin '%s' is already registered", datasourceChannelFactory.getName()));
}
try {
loadDatasourceClient(datasourceChannelFactory.getName());
} catch (Exception e) {
throw PluginException.getInstance(format("Datasource Plugin '%s' is can not load .", datasourceChannelFactory.getName()));
}
}
private void loadDatasourceClient(String name) {
requireNonNull(name, "name is null");
DataSourceChannelFactory datasourceChannelFactory = datasourceClientFactoryMap.get(name);
checkState(datasourceChannelFactory != null, "datasource Plugin {} is not registered", name);
try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(datasourceChannelFactory.getClass().getClassLoader())) {
DataSourceChannel datasourceChannel = datasourceChannelFactory.create();
this.datasourceClientMap.put(name, datasourceChannel);
}
logger.info("-- Loaded datasource Plugin {} --", name);
}
@Override
public void installPlugin(DolphinSchedulerPlugin dolphinSchedulerPlugin) {
for (DataSourceChannelFactory datasourceChannelFactory : dolphinSchedulerPlugin.getDatasourceChannelFactorys()) {
logger.info("Registering datasource Plugin '{}'", datasourceChannelFactory.getName());
this.addDatasourceClientFactory(datasourceChannelFactory);
}
}
}

84
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JdbcDataSourceProvider.java

@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.provider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource;
/**
* Jdbc Data Source Provider
*/
public class JdbcDataSourceProvider {
private static final Logger logger = LoggerFactory.getLogger(JdbcDataSourceProvider.class);
public static DruidDataSource createJdbcDataSource(BaseConnectionParam properties) {
logger.info("Creating DruidDataSource pool for maxActive:{}", PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50));
DruidDataSource druidDataSource = new DruidDataSource();
druidDataSource.setDriverClassName(properties.getDriverClassName());
druidDataSource.setUrl(properties.getJdbcUrl());
druidDataSource.setUsername(properties.getUser());
druidDataSource.setPassword(PasswordUtils.decodePassword(properties.getPassword()));
druidDataSource.setMinIdle(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MIN_IDLE, 5));
druidDataSource.setMaxActive(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50));
druidDataSource.setTestOnBorrow(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_TEST_ON_BORROW, false));
if (properties.getProps() != null) {
properties.getProps().forEach(druidDataSource::addConnectionProperty);
}
logger.info("Creating DruidDataSource pool success.");
return druidDataSource;
}
/**
* @return One Session Jdbc DataSource
*/
public static DruidDataSource createOneSessionJdbcDataSource(BaseConnectionParam properties) {
logger.info("Creating OneSession DruidDataSource pool for maxActive:{}", PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50));
DruidDataSource druidDataSource = new DruidDataSource();
druidDataSource.setDriverClassName(properties.getDriverClassName());
druidDataSource.setUrl(properties.getJdbcUrl());
druidDataSource.setUsername(properties.getUser());
druidDataSource.setPassword(PasswordUtils.decodePassword(properties.getPassword()));
druidDataSource.setMinIdle(1);
druidDataSource.setMaxActive(1);
druidDataSource.setTestOnBorrow(true);
if (properties.getProps() != null) {
properties.getProps().forEach(druidDataSource::addConnectionProperty);
}
logger.info("Creating OneSession DruidDataSource pool success.");
return druidDataSource;
}
}

5
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/util/CommonUtils.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.util;
package org.apache.dolphinscheduler.plugin.datasource.api.utils;
import static org.apache.dolphinscheduler.spi.task.TaskConstants.HADOOP_SECURITY_AUTHENTICATION;
import static org.apache.dolphinscheduler.spi.task.TaskConstants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE;
@ -41,8 +41,6 @@ import java.io.IOException;
*/
public class CommonUtils {
public static final String resourceUploadPath = PropertyUtils.getString(TaskConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler");
private CommonUtils() {
throw new UnsupportedOperationException("Construct CommonUtils");
}
@ -130,6 +128,7 @@ public class CommonUtils {
* @return data hdfs path
*/
public static String getHdfsDataBasePath() {
String resourceUploadPath = PropertyUtils.getString(TaskConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler");
if ("/".equals(resourceUploadPath)) {
// if basepath is configured to /, the generated url may be //default/resources (with extra leading /)
return "";

43
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtil.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DatasourceUtil.java

@ -15,20 +15,21 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
import org.apache.dolphinscheduler.common.datasource.clickhouse.ClickHouseDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.db2.Db2DatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.hive.HiveDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.presto.PrestoDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.spark.SparkDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerDatasourceProcessor;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.sql.Connection;
package org.apache.dolphinscheduler.plugin.datasource.api.utils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickHouseDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2DatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSqlDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SqlServerDatasourceProcessor;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -77,14 +78,6 @@ public class DatasourceUtil {
return getDatasourceProcessor(dbType).createConnectionParams(connectionJson);
}
public static Connection getConnection(DbType dbType, ConnectionParam connectionParam) {
try {
return getDatasourceProcessor(dbType).getConnection(connectionParam);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public static String getJdbcUrl(DbType dbType, ConnectionParam baseConnectionParam) {
return getDatasourceProcessor(dbType).getJdbcUrl(baseConnectionParam);
}
@ -118,4 +111,10 @@ public class DatasourceUtil {
}
}
/**
* get datasource UniqueId
*/
public static String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) {
return getDatasourceProcessor(dbType).getDatasourceUniqueId(connectionParam, dbType);
}
}

2
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/datasource/PasswordUtils.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/PasswordUtils.java

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.datasource;
package org.apache.dolphinscheduler.plugin.datasource.api.utils;
import static org.apache.dolphinscheduler.spi.task.TaskConstants.DATASOURCE_ENCRYPTION_ENABLE;
import static org.apache.dolphinscheduler.spi.task.TaskConstants.DATASOURCE_ENCRYPTION_SALT;

100
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClientTest.java

@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.client;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.api.provider.JdbcDataSourceProvider;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import java.sql.Connection;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.jdbc.core.JdbcTemplate;
import com.alibaba.druid.pool.DruidDataSource;
@RunWith(PowerMockRunner.class)
@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient")
@PrepareForTest(value = {DruidDataSource.class, CommonDataSourceClient.class, JdbcDataSourceProvider.class, JdbcTemplate.class, Connection.class})
public class CommonDataSourceClientTest {
@Mock
private CommonDataSourceClient commonDataSourceClient;
@Test
public void testPreInit() {
PowerMockito.doNothing().when(commonDataSourceClient).preInit();
commonDataSourceClient.preInit();
Mockito.verify(commonDataSourceClient).preInit();
}
@Test
public void testCheckEnv() {
BaseConnectionParam baseConnectionParam = new MysqlConnectionParam();
PowerMockito.doNothing().when(commonDataSourceClient).checkEnv(Mockito.any(BaseConnectionParam.class));
commonDataSourceClient.checkEnv(baseConnectionParam);
Mockito.verify(commonDataSourceClient).checkEnv(Mockito.any(BaseConnectionParam.class));
PowerMockito.doNothing().when(commonDataSourceClient).checkValidationQuery(Mockito.any(BaseConnectionParam.class));
commonDataSourceClient.checkValidationQuery(baseConnectionParam);
Mockito.verify(commonDataSourceClient).checkValidationQuery(Mockito.any(BaseConnectionParam.class));
PowerMockito.doNothing().when(commonDataSourceClient).checkUser(Mockito.any(BaseConnectionParam.class));
commonDataSourceClient.checkUser(baseConnectionParam);
Mockito.verify(commonDataSourceClient).checkUser(Mockito.any(BaseConnectionParam.class));
PowerMockito.doNothing().when(commonDataSourceClient).setDefaultUsername(Mockito.any(BaseConnectionParam.class));
commonDataSourceClient.setDefaultUsername(baseConnectionParam);
Mockito.verify(commonDataSourceClient).setDefaultUsername(Mockito.any(BaseConnectionParam.class));
PowerMockito.doNothing().when(commonDataSourceClient).setDefaultPassword(Mockito.any(BaseConnectionParam.class));
commonDataSourceClient.setDefaultPassword(baseConnectionParam);
Mockito.verify(commonDataSourceClient).setDefaultPassword(Mockito.any(BaseConnectionParam.class));
}
@Test
public void testInitClient() {
BaseConnectionParam baseConnectionParam = new MysqlConnectionParam();
PowerMockito.doNothing().when(commonDataSourceClient).initClient(Mockito.any(BaseConnectionParam.class));
commonDataSourceClient.initClient(baseConnectionParam);
Mockito.verify(commonDataSourceClient).initClient(Mockito.any(BaseConnectionParam.class));
}
@Test
public void testCheckClient() {
PowerMockito.doNothing().when(this.commonDataSourceClient).checkClient();
this.commonDataSourceClient.checkClient();
Mockito.verify(this.commonDataSourceClient).checkClient();
}
@Test
public void testGetConnection() {
Connection connection = PowerMockito.mock(Connection.class);
PowerMockito.when(commonDataSourceClient.getConnection()).thenReturn(connection);
Assert.assertNotNull(commonDataSourceClient.getConnection());
}
}

26
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java

@ -15,33 +15,46 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.clickhouse;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import java.sql.DriverManager;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class ClickHouseDatasourceProcessorTest {
private ClickHouseDatasourceProcessor clickHouseDatasourceProcessor = new ClickHouseDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
Map<String, String> props = new HashMap<>();
props.put("serverTimezone", "utc");
ClickHouseDatasourceParamDTO clickhouseConnectionParam = new ClickHouseDatasourceParamDTO();
clickhouseConnectionParam.setUserName("user");
clickhouseConnectionParam.setPassword("password");
clickhouseConnectionParam.setHost("localhost");
clickhouseConnectionParam.setPort(8123);
clickhouseConnectionParam.setDatabase("default");
clickhouseConnectionParam.setOther(props);
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) clickHouseDatasourceProcessor
.createConnectionParams(clickhouseConnectionParam);
Assert.assertNotNull(connectionParams);
@ -80,4 +93,9 @@ public class ClickHouseDatasourceProcessorTest {
public void testGetDbType() {
Assert.assertEquals(DbType.CLICKHOUSE, clickHouseDatasourceProcessor.getDbType());
}
@Test
public void testGetValidationQuery() {
Assert.assertEquals(Constants.CLICKHOUSE_VALIDATION_QUERY, clickHouseDatasourceProcessor.getValidationQuery());
}
}

25
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DatasourceProcessorTest.java

@ -15,33 +15,46 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.db2;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import java.sql.DriverManager;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class Db2DatasourceProcessorTest {
private Db2DatasourceProcessor db2DatasourceProcessor = new Db2DatasourceProcessor();
@Test
public void testCreateConnectionParams() {
Map<String, String> props = new HashMap<>();
props.put("serverTimezone", "utc");
Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO();
db2DatasourceParamDTO.setUserName("root");
db2DatasourceParamDTO.setPassword("123456");
db2DatasourceParamDTO.setHost("localhost");
db2DatasourceParamDTO.setPort(5142);
db2DatasourceParamDTO.setDatabase("default");
db2DatasourceParamDTO.setOther(props);
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
Db2ConnectionParam connectionParams = (Db2ConnectionParam) db2DatasourceProcessor
.createConnectionParams(db2DatasourceParamDTO);
@ -80,4 +93,8 @@ public class Db2DatasourceProcessorTest {
Assert.assertEquals(DbType.DB2, db2DatasourceProcessor.getDbType());
}
@Test
public void testGetValidationQuery() {
Assert.assertEquals(Constants.DB2_VALIDATION_QUERY, db2DatasourceProcessor.getValidationQuery());
}
}

30
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDatasourceProcessorTest.java

@ -15,33 +15,48 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.hive;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.sql.DriverManager;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class HiveDatasourceProcessorTest {
private HiveDatasourceProcessor hiveDatasourceProcessor = new HiveDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
Map<String, String> props = new HashMap<>();
props.put("serverTimezone", "utc");
HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO();
hiveDataSourceParamDTO.setHost("localhost1,localhost2");
hiveDataSourceParamDTO.setPort(5142);
hiveDataSourceParamDTO.setUserName("default");
hiveDataSourceParamDTO.setDatabase("default");
hiveDataSourceParamDTO.setOther(props);
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
PowerMockito.mockStatic(CommonUtils.class);
PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(false);
HiveConnectionParam connectionParams = (HiveConnectionParam) hiveDatasourceProcessor
.createConnectionParams(hiveDataSourceParamDTO);
System.out.println(JSONUtils.toJsonString(connectionParams));
@ -76,4 +91,9 @@ public class HiveDatasourceProcessorTest {
public void testGetDbType() {
Assert.assertEquals(DbType.HIVE, hiveDatasourceProcessor.getDbType());
}
@Test
public void testGetValidationQuery() {
Assert.assertEquals(Constants.HIVE_VALIDATION_QUERY, hiveDatasourceProcessor.getValidationQuery());
}
}

37
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MysqlDatasourceProcessorTest.java

@ -15,35 +15,47 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.mysql;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.sql.DriverManager;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class MysqlDatasourceProcessorTest {
private MysqlDatasourceProcessor mysqlDatasourceProcessor = new MysqlDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
Map<String, String> props = new HashMap<>();
props.put("serverTimezone", "utc");
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO();
mysqlDatasourceParamDTO.setUserName("root");
mysqlDatasourceParamDTO.setPassword("123456");
mysqlDatasourceParamDTO.setHost("localhost");
mysqlDatasourceParamDTO.setPort(3306);
mysqlDatasourceParamDTO.setDatabase("default");
mysqlDatasourceParamDTO.setOther(props);
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
MysqlConnectionParam connectionParams = (MysqlConnectionParam) mysqlDatasourceProcessor
.createConnectionParams(mysqlDatasourceParamDTO);
System.out.println(JSONUtils.toJsonString(connectionParams));
@ -78,4 +90,17 @@ public class MysqlDatasourceProcessorTest {
public void testGetDbType() {
Assert.assertEquals(DbType.MYSQL, mysqlDatasourceProcessor.getDbType());
}
@Test
public void testGetValidationQuery() {
Assert.assertEquals(Constants.MYSQL_VALIDATION_QUERY, mysqlDatasourceProcessor.getValidationQuery());
}
@Test
public void testGetDatasourceUniqueId() {
MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam();
mysqlConnectionParam.setJdbcUrl("jdbc:mysql://localhost:3306/default");
mysqlConnectionParam.setUser("root");
Assert.assertEquals("mysql@root@jdbc:mysql://localhost:3306/default", mysqlDatasourceProcessor.getDatasourceUniqueId(mysqlConnectionParam, DbType.MYSQL));
}
}

29
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessorTest.java

@ -15,28 +15,38 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.oracle;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbConnectType;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import java.sql.DriverManager;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class OracleDatasourceProcessorTest {
private OracleDatasourceProcessor oracleDatasourceProcessor = new OracleDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
Map<String, String> props = new HashMap<>();
props.put("serverTimezone", "utc");
OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO();
oracleDatasourceParamDTO.setConnectType(DbConnectType.ORACLE_SID);
oracleDatasourceParamDTO.setHost("localhost");
@ -44,7 +54,9 @@ public class OracleDatasourceProcessorTest {
oracleDatasourceParamDTO.setUserName("root");
oracleDatasourceParamDTO.setPassword("123456");
oracleDatasourceParamDTO.setDatabase("default");
oracleDatasourceParamDTO.setOther(props);
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
OracleConnectionParam connectionParams = (OracleConnectionParam) oracleDatasourceProcessor
.createConnectionParams(oracleDatasourceParamDTO);
Assert.assertNotNull(connectionParams);
@ -80,4 +92,9 @@ public class OracleDatasourceProcessorTest {
public void getDbType() {
Assert.assertEquals(DbType.ORACLE, oracleDatasourceProcessor.getDbType());
}
@Test
public void testGetValidationQuery() {
Assert.assertEquals(Constants.ORACLE_VALIDATION_QUERY, oracleDatasourceProcessor.getValidationQuery());
}
}

27
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java

@ -15,34 +15,46 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.postgresql;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import java.sql.DriverManager;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class PostgreSqlDatasourceProcessorTest {
private PostgreSqlDatasourceProcessor postgreSqlDatasourceProcessor = new PostgreSqlDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
Map<String, String> props = new HashMap<>();
props.put("serverTimezone", "utc");
PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO();
postgreSqlDatasourceParamDTO.setUserName("root");
postgreSqlDatasourceParamDTO.setPassword("123456");
postgreSqlDatasourceParamDTO.setHost("localhost");
postgreSqlDatasourceParamDTO.setPort(3308);
postgreSqlDatasourceParamDTO.setDatabase("default");
postgreSqlDatasourceParamDTO.setOther(props);
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) postgreSqlDatasourceProcessor
.createConnectionParams(postgreSqlDatasourceParamDTO);
Assert.assertEquals("jdbc:postgresql://localhost:3308", connectionParams.getAddress());
@ -80,4 +92,9 @@ public class PostgreSqlDatasourceProcessorTest {
public void testGetDbType() {
Assert.assertEquals(DbType.POSTGRESQL, postgreSqlDatasourceProcessor.getDbType());
}
@Test
public void testGetValidationQuery() {
Assert.assertEquals(Constants.POSTGRESQL_VALIDATION_QUERY, postgreSqlDatasourceProcessor.getValidationQuery());
}
}

27
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDatasourceProcessorTest.java

@ -15,34 +15,46 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.presto;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import java.sql.DriverManager;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class PrestoDatasourceProcessorTest {
private PrestoDatasourceProcessor prestoDatasourceProcessor = new PrestoDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
Map<String, String> props = new HashMap<>();
props.put("serverTimezone", "utc");
PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO();
prestoDatasourceParamDTO.setHost("localhost");
prestoDatasourceParamDTO.setPort(1234);
prestoDatasourceParamDTO.setDatabase("default");
prestoDatasourceParamDTO.setUserName("root");
prestoDatasourceParamDTO.setPassword("123456");
prestoDatasourceParamDTO.setOther(props);
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
PrestoConnectionParam connectionParams = (PrestoConnectionParam) prestoDatasourceProcessor
.createConnectionParams(prestoDatasourceParamDTO);
Assert.assertEquals("jdbc:presto://localhost:1234", connectionParams.getAddress());
@ -78,4 +90,9 @@ public class PrestoDatasourceProcessorTest {
public void testGetDbType() {
Assert.assertEquals(DbType.PRESTO, prestoDatasourceProcessor.getDbType());
}
@Test
public void testGetValidationQuery() {
Assert.assertEquals(Constants.PRESTO_VALIDATION_QUERY, prestoDatasourceProcessor.getValidationQuery());
}
}

29
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDatasourceProcessorTest.java

@ -15,34 +15,48 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.spark;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import java.sql.DriverManager;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class SparkDatasourceProcessorTest {
private SparkDatasourceProcessor sparkDatasourceProcessor = new SparkDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
Map<String, String> props = new HashMap<>();
props.put("serverTimezone", "utc");
SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO();
sparkDatasourceParamDTO.setUserName("root");
sparkDatasourceParamDTO.setPassword("12345");
sparkDatasourceParamDTO.setHost("localhost1,localhost2");
sparkDatasourceParamDTO.setPort(1234);
sparkDatasourceParamDTO.setDatabase("default");
sparkDatasourceParamDTO.setOther(props);
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
PowerMockito.mockStatic(CommonUtils.class);
PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(false);
SparkConnectionParam connectionParams = (SparkConnectionParam) sparkDatasourceProcessor
.createConnectionParams(sparkDatasourceParamDTO);
Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234", connectionParams.getAddress());
@ -77,4 +91,9 @@ public class SparkDatasourceProcessorTest {
public void testGetDbType() {
Assert.assertEquals(DbType.SPARK, sparkDatasourceProcessor.getDbType());
}
@Test
public void testGetValidationQuery() {
Assert.assertEquals(Constants.HIVE_VALIDATION_QUERY, sparkDatasourceProcessor.getValidationQuery());
}
}

29
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SqlServerDatasourceProcessorTest.java

@ -15,35 +15,47 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.sqlserver;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.sql.DriverManager;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
@PrepareForTest({Class.class, DriverManager.class, DatasourceUtil.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class SqlServerDatasourceProcessorTest {
private SqlServerDatasourceProcessor sqlServerDatasourceProcessor = new SqlServerDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
Map<String, String> props = new HashMap<>();
props.put("serverTimezone", "utc");
SqlServerDatasourceParamDTO sqlServerDatasourceParamDTO = new SqlServerDatasourceParamDTO();
sqlServerDatasourceParamDTO.setUserName("root");
sqlServerDatasourceParamDTO.setPassword("123456");
sqlServerDatasourceParamDTO.setDatabase("default");
sqlServerDatasourceParamDTO.setHost("localhost");
sqlServerDatasourceParamDTO.setPort(1234);
sqlServerDatasourceParamDTO.setOther(props);
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
SqlServerConnectionParam connectionParams = (SqlServerConnectionParam) sqlServerDatasourceProcessor
.createConnectionParams(sqlServerDatasourceParamDTO);
Assert.assertEquals("jdbc:sqlserver://localhost:1234", connectionParams.getAddress());
@ -78,4 +90,9 @@ public class SqlServerDatasourceProcessorTest {
public void testGetDbType() {
Assert.assertEquals(DbType.SQLSERVER, sqlServerDatasourceProcessor.getDbType());
}
@Test
public void testGetValidationQuery() {
Assert.assertEquals(Constants.SQLSERVER_VALIDATION_QUERY, sqlServerDatasourceProcessor.getValidationQuery());
}
}

52
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JdbcDataSourceProviderTest.java

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.provider;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlConnectionParam;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import com.alibaba.druid.pool.DruidDataSource;
@RunWith(PowerMockRunner.class)
@PrepareForTest(value = {DruidDataSource.class, JdbcDataSourceProvider.class})
public class JdbcDataSourceProviderTest {
@Test
public void testCreateJdbcDataSource() {
PowerMockito.mockStatic(JdbcDataSourceProvider.class);
DruidDataSource druidDataSource = PowerMockito.mock(DruidDataSource.class);
PowerMockito.when(JdbcDataSourceProvider.createJdbcDataSource(Mockito.any())).thenReturn(druidDataSource);
Assert.assertNotNull(JdbcDataSourceProvider.createJdbcDataSource(new MysqlConnectionParam()));
}
@Test
public void testCreateOneSessionJdbcDataSource() {
PowerMockito.mockStatic(JdbcDataSourceProvider.class);
DruidDataSource druidDataSource = PowerMockito.mock(DruidDataSource.class);
PowerMockito.when(JdbcDataSourceProvider.createOneSessionJdbcDataSource(Mockito.any())).thenReturn(druidDataSource);
Assert.assertNotNull(JdbcDataSourceProvider.createOneSessionJdbcDataSource(new MysqlConnectionParam()));
}
}

124
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtilsTest.java

@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.utils;
import static org.apache.dolphinscheduler.spi.task.TaskConstants.DATASOURCE_ENCRYPTION_ENABLE;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@SuppressStaticInitializationFor("org.apache.dolphinscheduler.spi.utils.PropertyUtils")
@PrepareForTest(value = {PropertyUtils.class, UserGroupInformation.class, CommonUtils.class, PasswordUtils.class})
public class CommonUtilsTest {
@Test
public void testGetKerberosStartupState() {
PowerMockito.mockStatic(CommonUtils.class);
PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(false);
boolean kerberosStartupState = CommonUtils.getKerberosStartupState();
Assert.assertFalse(kerberosStartupState);
PowerMockito.mockStatic(PropertyUtils.class);
PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS");
PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, true)).thenReturn(Boolean.TRUE);
kerberosStartupState = CommonUtils.getKerberosStartupState();
Assert.assertFalse(kerberosStartupState);
}
@Test
public void testLoadKerberosConf() {
try {
PowerMockito.mockStatic(PropertyUtils.class);
PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS");
PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE);
PowerMockito.when(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)).thenReturn("/opt/krb5.conf");
PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)).thenReturn("hdfs-mycluster@ESZ.COM");
PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)).thenReturn("/opt/hdfs.headless.keytab");
PowerMockito.mockStatic(UserGroupInformation.class);
boolean result = CommonUtils.loadKerberosConf(new Configuration());
Assert.assertTrue(result);
CommonUtils.loadKerberosConf(null, null, null);
} catch (Exception e) {
Assert.fail("load Kerberos Conf failed");
}
}
@Test
public void encodePassword() {
PowerMockito.mockStatic(PropertyUtils.class);
PowerMockito.when(PropertyUtils.getBoolean(DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.TRUE);
Assert.assertEquals("", PasswordUtils.encodePassword(""));
Assert.assertEquals("bnVsbE1USXpORFUy", PasswordUtils.encodePassword("123456"));
Assert.assertEquals("bnVsbElWRkJXbGhUVjBBPQ==", PasswordUtils.encodePassword("!QAZXSW@"));
Assert.assertEquals("bnVsbE5XUm1aMlZ5S0VBPQ==", PasswordUtils.encodePassword("5dfger(@"));
PowerMockito.when(PropertyUtils.getBoolean(DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.FALSE);
Assert.assertEquals("", PasswordUtils.encodePassword(""));
Assert.assertEquals("123456", PasswordUtils.encodePassword("123456"));
Assert.assertEquals("!QAZXSW@", PasswordUtils.encodePassword("!QAZXSW@"));
Assert.assertEquals("5dfger(@", PasswordUtils.encodePassword("5dfger(@"));
}
@Test
public void decodePassword() {
PowerMockito.mockStatic(PropertyUtils.class);
PowerMockito.when(PropertyUtils.getBoolean(DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.TRUE);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.decodePassword("bnVsbE1USXpORFUy")).thenReturn("123456");
PowerMockito.when(PasswordUtils.decodePassword("bnVsbElWRkJXbGhUVjBBPQ==")).thenReturn("!QAZXSW@");
PowerMockito.when(PasswordUtils.decodePassword("bnVsbE5XUm1aMlZ5S0VBPQ==")).thenReturn("5dfger(@");
Assert.assertEquals(null, PasswordUtils.decodePassword(""));
Assert.assertEquals("123456", PasswordUtils.decodePassword("bnVsbE1USXpORFUy"));
Assert.assertEquals("!QAZXSW@", PasswordUtils.decodePassword("bnVsbElWRkJXbGhUVjBBPQ=="));
Assert.assertEquals("5dfger(@", PasswordUtils.decodePassword("bnVsbE5XUm1aMlZ5S0VBPQ=="));
PowerMockito.when(PropertyUtils.getBoolean(DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.FALSE);
PowerMockito.when(PasswordUtils.decodePassword("123456")).thenReturn("123456");
PowerMockito.when(PasswordUtils.decodePassword("!QAZXSW@")).thenReturn("!QAZXSW@");
PowerMockito.when(PasswordUtils.decodePassword("5dfger(@")).thenReturn("5dfger(@");
Assert.assertEquals(null, PasswordUtils.decodePassword(""));
Assert.assertEquals("123456", PasswordUtils.decodePassword("123456"));
Assert.assertEquals("!QAZXSW@", PasswordUtils.decodePassword("!QAZXSW@"));
Assert.assertEquals("5dfger(@", PasswordUtils.decodePassword("5dfger(@"));
}
}

42
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DatasourceUtilTest.java

@ -15,19 +15,20 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
package org.apache.dolphinscheduler.plugin.datasource.api.utils;
import java.util.HashMap;
import java.util.Map;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MysqlDatasourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
@ -38,7 +39,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class, MysqlDatasourceProcessor.class})
@PrepareForTest({Class.class, DriverManager.class, MysqlDatasourceProcessor.class, DataSourceClientProvider.class, PasswordUtils.class, CommonUtils.class})
public class DatasourceUtilTest {
@Test
@ -63,6 +64,10 @@ public class DatasourceUtilTest {
mysqlDatasourceParamDTO.setUserName("root");
mysqlDatasourceParamDTO.setPort(3306);
mysqlDatasourceParamDTO.setPassword("123456");
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("123456");
PowerMockito.mockStatic(CommonUtils.class);
PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(false);
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO);
Assert.assertNotNull(connectionParam);
}
@ -80,18 +85,21 @@ public class DatasourceUtilTest {
}
@Test
public void testGetConnection() throws ClassNotFoundException, SQLException {
PowerMockito.mockStatic(Class.class);
PowerMockito.when(Class.forName(Mockito.any())).thenReturn(null);
PowerMockito.mockStatic(DriverManager.class);
PowerMockito.when(DriverManager.getConnection(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(null);
public void testGetConnection() {
PowerMockito.mockStatic(DataSourceClientProvider.class);
DataSourceClientProvider clientProvider = PowerMockito.mock(DataSourceClientProvider.class);
PowerMockito.when(DataSourceClientProvider.getInstance()).thenReturn(clientProvider);
Connection connection = PowerMockito.mock(Connection.class);
PowerMockito.when(clientProvider.getConnection(Mockito.any(), Mockito.any())).thenReturn(connection);
MysqlConnectionParam connectionParam = new MysqlConnectionParam();
connectionParam.setUser("root");
connectionParam.setPassword("123456");
Connection connection = DatasourceUtil.getConnection(DbType.MYSQL, connectionParam);
connection = DataSourceClientProvider.getInstance().getConnection(DbType.MYSQL, connectionParam);
Assert.assertNull(connection);
Assert.assertNotNull(connection);
}

114
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml

@ -0,0 +1,114 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dolphinscheduler-datasource-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dolphinscheduler-datasource-clickhouse</artifactId>
<name>${project.artifactId}</name>
<!-- can be load as a datasource Plugin when development and run server in IDE -->
<packaging>dolphinscheduler-plugin</packaging>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<exclusions>
<exclusion>
<artifactId>jaxb-api</artifactId>
<groupId>javax.xml.bind</groupId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</exclusion>
</exclusions>
<version>${clickhouse.jdbc.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<type>jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jacoco</groupId>
<artifactId>org.jacoco.agent</artifactId>
<classifier>runtime</classifier>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>dolphinscheduler-datasource-clickhouse-${project.version}</finalName>
</build>
</project>

20
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2ConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannel.java

@ -15,20 +15,16 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.db2;
package org.apache.dolphinscheduler.plugin.datasource.clickhouse;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
import org.apache.dolphinscheduler.spi.datasource.DataSourceClient;
public class ClickhouseDataSourceChannel implements DataSourceChannel {
public class Db2ConnectionParam extends BaseConnectionParam {
@Override
public String toString() {
return "Db2ConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ '}';
public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam) {
return new ClickhouseDataSourceClient(baseConnectionParam);
}
}

33
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelFactory.java

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.clickhouse;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
public class ClickhouseDataSourceChannelFactory implements DataSourceChannelFactory {
@Override
public String getName() {
return "clickhouse";
}
@Override
public DataSourceChannel create() {
return new ClickhouseDataSourceChannel();
}
}

29
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceClient.java

@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.clickhouse;
import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
public class ClickhouseDataSourceClient extends CommonDataSourceClient {
public ClickhouseDataSourceClient(BaseConnectionParam baseConnectionParam) {
super(baseConnectionParam);
}
}

30
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourcePlugin.java

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.clickhouse;
import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
import com.google.common.collect.ImmutableList;
public class ClickhouseDataSourcePlugin implements DolphinSchedulerPlugin {
@Override
public Iterable<DataSourceChannelFactory> getDatasourceChannelFactorys() {
return ImmutableList.of(new ClickhouseDataSourceChannelFactory());
}
}

33
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelFactoryTest.java

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.clickhouse;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
import org.junit.Assert;
import org.junit.Test;
public class ClickhouseDataSourceChannelFactoryTest {
@Test
public void testCreate() {
ClickhouseDataSourceChannelFactory sourceChannelFactory = new ClickhouseDataSourceChannelFactory();
DataSourceChannel dataSourceChannel = sourceChannelFactory.create();
Assert.assertNotNull(dataSourceChannel);
}
}

43
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourceChannelTest.java

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.clickhouse;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickhouseConnectionParam;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient")
@PrepareForTest({ClickhouseDataSourceClient.class, ClickhouseDataSourceChannel.class})
public class ClickhouseDataSourceChannelTest {
@Test
public void testCreateDataSourceClient() {
ClickhouseDataSourceChannel sourceChannel = PowerMockito.mock(ClickhouseDataSourceChannel.class);
ClickhouseDataSourceClient dataSourceClient = PowerMockito.mock(ClickhouseDataSourceClient.class);
PowerMockito.when(sourceChannel.createDataSourceClient(Mockito.any())).thenReturn(dataSourceClient);
Assert.assertNotNull(sourceChannel.createDataSourceClient(new ClickhouseConnectionParam()));
}
}

37
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickhouseDataSourcePluginTest.java

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.clickhouse;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
import org.junit.Assert;
import org.junit.Test;
public class ClickhouseDataSourcePluginTest {
private ClickhouseDataSourcePlugin dataSourcePlugin = new ClickhouseDataSourcePlugin();
@Test
public void testGetDatasourceChannelFactorys() {
Iterable<DataSourceChannelFactory> channelFactorys = dataSourcePlugin.getDatasourceChannelFactorys();
for (DataSourceChannelFactory dataSourceChannelFactory : channelFactorys) {
Assert.assertTrue(dataSourceChannelFactory instanceof ClickhouseDataSourceChannelFactory);
}
}
}

91
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/pom.xml

@ -0,0 +1,91 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dolphinscheduler-datasource-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dolphinscheduler-datasource-db2</artifactId>
<name>${project.artifactId}</name>
<!-- can be load as a datasource Plugin when development and run server in IDE -->
<packaging>dolphinscheduler-plugin</packaging>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<type>jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jacoco</groupId>
<artifactId>org.jacoco.agent</artifactId>
<classifier>runtime</classifier>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>dolphinscheduler-datasource-db2-${project.version}</finalName>
</build>
</project>

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save