From 36d60a564adef2bfec49705f82422f38591bee9f Mon Sep 17 00:00:00 2001 From: Kerwin <37063904+zhuangchong@users.noreply.github.com> Date: Tue, 7 Sep 2021 15:04:18 +0800 Subject: [PATCH] [Feature][Task]Task plugin spi (#6116) --- .../common/utils/DateUtils.java | 32 + .../worker/runner/TaskExecuteThread.java | 5 +- dolphinscheduler-spi/pom.xml | 226 ++++++- .../spi/enums/CommandType.java | 84 +++ .../dolphinscheduler/spi/enums/DataType.java | 37 ++ .../spi/enums/DbConnectType.java | 42 ++ .../dolphinscheduler/spi/enums/DbType.java | 59 ++ .../dolphinscheduler/spi/enums/Flag.java | 51 ++ .../spi/enums/ResUploadType.java | 30 + .../spi/enums/TaskTimeoutStrategy.java | 58 ++ .../spi/task/AbstractTask.java | 2 + .../dolphinscheduler/spi/task/Property.java | 2 +- .../spi/task/TaskChannel.java | 2 + .../spi/task/TaskConstants.java | 241 +++++++ .../spi/task/UdfFuncBean.java | 188 ++++++ .../AbstractDatasourceProcessor.java | 79 +++ .../task/datasource/BaseConnectionParam.java | 108 +++ .../datasource/BaseDataSourceParamDTO.java | 161 +++++ .../datasource/BaseHdfsConnectionParam.java | 57 ++ .../BaseHdfsDatasourceParamDTO.java | 61 ++ .../spi/task/datasource/ConnectionParam.java | 26 + .../task/datasource/DatasourceProcessor.java | 81 +++ .../spi/task/datasource/DatasourceUtil.java | 121 ++++ .../spi/task/datasource/HiveConfUtils.java | 87 +++ .../spi/task/datasource/PasswordUtils.java | 88 +++ .../ClickHouseDatasourceParamDTO.java | 41 ++ .../ClickHouseDatasourceProcessor.java | 131 ++++ .../clickhouse/ClickhouseConnectionParam.java | 34 + .../datasource/db2/Db2ConnectionParam.java | 34 + .../datasource/db2/Db2DatasourceParamDTO.java | 43 ++ .../db2/Db2DatasourceProcessor.java | 132 ++++ .../datasource/hive/HiveConnectionParam.java | 38 ++ .../hive/HiveDataSourceParamDTO.java | 45 ++ .../hive/HiveDatasourceProcessor.java | 192 ++++++ .../mysql/MysqlConnectionParam.java | 35 + .../mysql/MysqlDatasourceParamDTO.java | 43 ++ .../mysql/MysqlDatasourceProcessor.java | 176 +++++ .../oracle/OracleConnectionParam.java | 47 ++ .../oracle/OracleDatasourceParamDTO.java | 55 ++ .../oracle/OracleDatasourceProcessor.java | 149 +++++ .../postgresql/PostgreSqlConnectionParam.java | 34 + .../PostgreSqlDatasourceParamDTO.java | 41 ++ .../PostgreSqlDatasourceProcessor.java | 132 ++++ .../presto/PrestoConnectionParam.java | 34 + .../presto/PrestoDatasourceParamDTO.java | 43 ++ .../presto/PrestoDatasourceProcessor.java | 134 ++++ .../spark/SparkConnectionParam.java | 38 ++ .../spark/SparkDatasourceParamDTO.java | 45 ++ .../spark/SparkDatasourceProcessor.java | 161 +++++ .../sqlserver/SqlServerConnectionParam.java | 34 + .../SqlServerDatasourceParamDTO.java | 43 ++ .../SqlServerDatasourceProcessor.java | 129 ++++ .../task/paramparser/BusinessTimeUtils.java | 78 +++ .../spi/task/paramparser/ParamUtils.java | 158 +++++ .../spi/task/paramparser/ParameterUtils.java | 269 ++++++++ .../task/paramparser/PlaceholderUtils.java | 103 +++ .../PropertyPlaceholderHelper.java | 255 +++++++ .../paramparser/TimePlaceholderUtils.java | 570 ++++++++++++++++ .../spi/task/request/DataxTaskRequest.java | 115 ++++ .../task/request/ProcedureTaskRequest.java | 45 ++ .../spi/task/request/SQLTaskRequest.java | 80 +++ .../spi/task/request/SqoopTaskRequest.java | 115 ++++ .../spi/task/{ => request}/TaskRequest.java | 16 +- .../spi/utils/CommonUtils.java | 104 +++ .../dolphinscheduler/spi/utils/Constants.java | 20 + .../dolphinscheduler/spi/utils/DateUtils.java | 622 ++++++++++++++++++ .../spi/utils/PropertyUtils.java | 260 ++++++++ .../spi/utils/StringUtils.java | 195 +++++- .../task/api/AbstractCommandExecutor.java | 2 +- .../plugin/task/api/AbstractTaskExecutor.java | 2 +- .../plugin/task/api/AbstractYarnTask.java | 9 +- .../plugin/task/api/ProcessUtils.java | 2 +- .../plugin/task/api/ShellCommandExecutor.java | 2 +- .../api/TaskExecutionContextCacheManager.java | 2 +- .../dolphinscheduler-task-datax/pom.xml | 12 +- .../plugin/task/datax/DataxParameters.java | 251 +++++++ .../plugin/task/datax/DataxTask.java | 570 ++++++++++++++++ .../plugin/task/datax/DataxTaskChannel.java | 36 + .../task/datax/DataxTaskChannelFactory.java | 42 ++ .../plugin/task/datax/DataxTaskPlugin.java | 31 + .../plugin/task/datax/DataxUtils.java | 136 ++++ .../dolphinscheduler-task-flink/pom.xml | 22 +- .../plugin/task/flink/FlinkTask.java | 87 +-- .../plugin/task/flink/FlinkTaskChannel.java | 2 +- .../dolphinscheduler-task-http/pom.xml | 12 +- .../plugin/task/http/HttpMethod.java | 18 +- .../plugin/task/http/HttpParameters.java | 5 +- .../plugin/task/http/HttpParametersType.java | 1 + .../plugin/task/http/HttpProperty.java | 192 +++--- .../plugin/task/http/HttpTask.java | 269 +++++++- .../plugin/task/http/HttpTaskChannel.java | 4 +- .../task/http/HttpTaskChannelFactory.java | 5 +- .../plugin/task/http/HttpTaskConstants.java} | 8 +- .../plugin/task/http/HttpTaskPlugin.java | 3 +- .../dolphinscheduler-task-mr/pom.xml | 5 + .../plugin/task/mr/MapReduceArgsUtils.java | 88 +++ .../plugin/task/mr/MapReduceParameters.java | 159 +++++ .../plugin/task/mr/MapReduceTask.java | 134 ++++ .../plugin/task/mr/MapReduceTaskChannel.java | 34 + .../task/mr/MapReduceTaskChannelFactory.java | 42 ++ .../task/mr/MapReduceTaskConstants.java | 36 + .../plugin/task/mr/MapReduceTaskPlugin.java | 31 + .../plugin/task/mr/ProgramType.java | 30 + .../dolphinscheduler-task-procedure/pom.xml | 7 +- .../task/procedure/ProcedureParameters.java | 89 +++ .../plugin/task/procedure/ProcedureTask.java | 330 ++++++++++ .../task/procedure/ProcedureTaskChannel.java | 36 + .../ProcedureTaskChannelFactory.java | 42 ++ .../task/procedure/ProcedureTaskPlugin.java | 31 + .../dolphinscheduler-task-python/pom.xml | 22 +- .../task/python/PythonCommandExecutor.java | 2 +- .../plugin/task/python/PythonTask.java | 5 +- .../plugin/task/python/PythonTaskChannel.java | 2 +- .../plugin/task/shell/ShellTask.java | 4 +- .../plugin/task/shell/ShellTaskChannel.java | 6 +- .../dolphinscheduler-task-spark/pom.xml | 24 +- .../plugin/task/spark/SparkTask.java | 53 +- .../plugin/task/spark/SparkTaskChannel.java | 5 +- .../task/spark/SparkTaskChannelFactory.java | 2 +- .../plugin/task/spark/SparkTaskPlugin.java | 1 + .../dolphinscheduler-task-sqoop/pom.xml | 4 + .../plugin/task/sqoop/SqoopConstants.java | 75 +++ .../plugin/task/sqoop/SqoopJobType.java | 40 ++ .../plugin/task/sqoop/SqoopQueryType.java | 41 ++ .../plugin/task/sqoop/SqoopTask.java | 93 +++ .../plugin/task/sqoop/SqoopTaskChannel.java | 36 + .../task/sqoop/SqoopTaskChannelFactory.java | 42 ++ .../plugin/task/sqoop/SqoopTaskPlugin.java | 31 + .../task/sqoop/generator/CommonGenerator.java | 88 +++ .../sqoop/generator/ISourceGenerator.java | 36 + .../sqoop/generator/ITargetGenerator.java | 36 + .../sqoop/generator/SqoopJobGenerator.java | 124 ++++ .../sources/HdfsSourceGenerator.java | 64 ++ .../sources/HiveSourceGenerator.java | 77 +++ .../sources/MysqlSourceGenerator.java | 147 +++++ .../targets/HdfsTargetGenerator.java | 93 +++ .../targets/HiveTargetGenerator.java | 101 +++ .../targets/MysqlTargetGenerator.java | 123 ++++ .../task/sqoop/parameter/SqoopParameters.java | 204 ++++++ .../sources/SourceHdfsParameter.java | 37 ++ .../sources/SourceHiveParameter.java | 73 ++ .../sources/SourceMysqlParameter.java | 137 ++++ .../targets/TargetHdfsParameter.java | 98 +++ .../targets/TargetHiveParameter.java | 121 ++++ .../targets/TargetMysqlParameter.java | 133 ++++ .../plugin/task/tis/TISTask.java | 6 +- .../plugin/task/tis/TISTaskChannel.java | 2 +- .../plugin/task/tis/TISTaskTest.java | 2 +- dolphinscheduler-task-plugin/pom.xml | 10 +- pom.xml | 2 +- 150 files changed, 11732 insertions(+), 285 deletions(-) create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DataType.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbConnectType.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResUploadType.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/TaskTimeoutStrategy.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/UdfFuncBean.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/AbstractDatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseDataSourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseHdfsConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseHdfsDatasourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/ConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/DatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/DatasourceUtil.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/HiveConfUtils.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/PasswordUtils.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickHouseDatasourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickhouseConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2ConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2DatasourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2DatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveDataSourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveDatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlDatasourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlDatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleDatasourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleDatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlDatasourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoDatasourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoDatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkDatasourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkDatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerConnectionParam.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerDatasourceParamDTO.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerDatasourceProcessor.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/BusinessTimeUtils.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/ParamUtils.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/ParameterUtils.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/PlaceholderUtils.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/PropertyPlaceholderHelper.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/TimePlaceholderUtils.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/DataxTaskRequest.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/ProcedureTaskRequest.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/SQLTaskRequest.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/SqoopTaskRequest.java rename dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/{ => request}/TaskRequest.java (95%) create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/CommonUtils.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java create mode 100644 dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannel.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannelFactory.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskPlugin.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java rename dolphinscheduler-task-plugin/{dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskConstants.java => dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskConstants.java} (83%) create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceArgsUtils.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannelFactory.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskConstants.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskPlugin.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/ProgramType.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannel.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannelFactory.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskPlugin.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopQueryType.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannel.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannelFactory.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskPlugin.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/ISourceGenerator.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/ITargetGenerator.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MysqlSourceGenerator.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MysqlTargetGenerator.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceHdfsParameter.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceHiveParameter.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceMysqlParameter.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetHdfsParameter.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetHiveParameter.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetMysqlParameter.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java index ec929112a7..5a6a704e02 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java @@ -483,6 +483,38 @@ public class DateUtils { Constants.YYYY_MM_DD_HH_MM_SS); } + public static Date addYears(Date date, int amount) { + return add(date, 1, amount); + } + + public static Date addMonths(Date date, int amount) { + return add(date, 2, amount); + } + + public static Date addWeeks(Date date, int amount) { + return add(date, 3, amount); + } + + public static Date addDays(Date date, int amount) { + return add(date, 5, amount); + } + + public static Date addHours(Date date, int amount) { + return add(date, 11, amount); + } + + public static Date addMinutes(Date date, int amount) { + return add(date, 12, amount); + } + + public static Date addSeconds(Date date, int amount) { + return add(date, 13, amount); + } + + public static Date addMilliseconds(Date date, int amount) { + return add(date, 14, amount); + } + /** * get date * diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index 6376f3a4cc..56744738df 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -41,7 +41,7 @@ import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.spi.task.AbstractTask; import org.apache.dolphinscheduler.spi.task.TaskChannel; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.commons.collections.MapUtils; @@ -57,9 +57,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import org.apache.dolphinscheduler.spi.task.AbstractTask; -import org.apache.dolphinscheduler.spi.task.TaskChannel; -import org.apache.dolphinscheduler.spi.task.TaskRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-spi/pom.xml b/dolphinscheduler-spi/pom.xml index 79b7a583b9..b96161e1b9 100644 --- a/dolphinscheduler-spi/pom.xml +++ b/dolphinscheduler-spi/pom.xml @@ -56,7 +56,11 @@ commons-beanutils provided - + + commons-codec + commons-codec + provided + org.slf4j slf4j-api @@ -94,6 +98,226 @@ resolver provided + + + org.apache.hive + hive-jdbc + provided + + + slf4j-log4j12 + org.slf4j + + + org.eclipse.jetty.aggregate + jetty-all + + + + org.apache.ant + ant + + + io.dropwizard.metrics + metrics-json + + + io.dropwizard.metrics + metrics-jvm + + + com.github.joshelser + dropwizard-metrics-hadoop-metrics2-reporter + + + + io.netty + netty-all + + + com.google.code.gson + gson + + + com.google.code.findbugs + jsr305 + + + io.dropwizard.metrics + metrics-core + + + javax.servlet + servlet-api + + + org.apache.avro + avro + + + org.apache.commons + commons-compress + + + org.apache.curator + curator-client + + + org.apache.hadoop + hadoop-auth + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.apache.hadoop + hadoop-yarn-api + + + + org.apache.zookeeper + zookeeper + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + com.google.protobuf + protobuf-java + + + + org.json + json + + + log4j-slf4j-impl + org.apache.logging.log4j + + + javax.servlet + org.eclipse.jetty.orbit + + + servlet-api-2.5 + org.mortbay.jetty + + + jasper-runtime + tomcat + + + slider-core + org.apache.slider + + + hbase-server + org.apache.hbase + + + jersey-client + com.sun.jersey + + + jersey-core + com.sun.jersey + + + jersey-json + com.sun.jersey + + + jersey-server + com.sun.jersey + + + jersey-guice + com.sun.jersey.contribs + + + hbase-common + org.apache.hbase + + + hbase-hadoop2-compat + org.apache.hbase + + + hbase-client + org.apache.hbase + + + hbase-hadoop-compat + org.apache.hbase + + + tephra-hbase-compat-1.0 + co.cask.tephra + + + jaxb-api + javax.xml.bind + + + hive-llap-client + org.apache.hive + + + hive-llap-common + org.apache.hive + + + hive-llap-server + org.apache.hive + + + tephra-core + co.cask.tephra + + + ant + ant + + + stringtemplate + org.antlr + + + antlr-runtime + org.antlr + + + hive-shims + org.apache.hive + + + jsp-api + javax.servlet + + + log4j-api + org.apache.logging.log4j + + + log4j-core + org.apache.logging.log4j + + + log4j-web + org.apache.logging.log4j + + + jasper-compiler + tomcat + + + \ No newline at end of file diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java new file mode 100644 index 0000000000..f3aa8ff6d2 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.enums; + +import java.util.HashMap; +import java.util.Map; + +/** + * command types + */ +public enum CommandType { + + /** + * command types + * 0 start a new process + * 1 start a new process from current nodes + * 2 recover tolerance fault process + * 3 recover suspended process + * 4 start process from failure task nodes + * 5 complement data + * 6 start a new process from scheduler + * 7 repeat running a process + * 8 pause a process + * 9 stop a process + * 10 recover waiting thread + */ + START_PROCESS(0, "start a new process"), + START_CURRENT_TASK_PROCESS(1, "start a new process from current nodes"), + RECOVER_TOLERANCE_FAULT_PROCESS(2, "recover tolerance fault process"), + RECOVER_SUSPENDED_PROCESS(3, "recover suspended process"), + START_FAILURE_TASK_PROCESS(4, "start process from failure task nodes"), + COMPLEMENT_DATA(5, "complement data"), + SCHEDULER(6, "start a new process from scheduler"), + REPEAT_RUNNING(7, "repeat running a process"), + PAUSE(8, "pause a process"), + STOP(9, "stop a process"), + RECOVER_WAITING_THREAD(10, "recover waiting thread"); + + CommandType(int code, String descp) { + this.code = code; + this.descp = descp; + } + + private final int code; + private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } + + private static final Map COMMAND_TYPE_MAP = new HashMap<>(); + + static { + for (CommandType commandType : CommandType.values()) { + COMMAND_TYPE_MAP.put(commandType.code,commandType); + } + } + + public static CommandType of(Integer status) { + if (COMMAND_TYPE_MAP.containsKey(status)) { + return COMMAND_TYPE_MAP.get(status); + } + throw new IllegalArgumentException("invalid status : " + status); + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DataType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DataType.java new file mode 100644 index 0000000000..f4787e5fec --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DataType.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.enums; + +/** + * data types in user define parameter + */ +public enum DataType { + /** + * 0 string + * 1 integer + * 2 long + * 3 float + * 4 double + * 5 date, "YYYY-MM-DD" + * 6 time, "HH:MM:SS" + * 7 time stamp + * 8 Boolean + * 9 list + */ + VARCHAR,INTEGER,LONG,FLOAT,DOUBLE,DATE,TIME,TIMESTAMP,BOOLEAN,LIST +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbConnectType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbConnectType.java new file mode 100644 index 0000000000..72b0322da7 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbConnectType.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.enums; + +public enum DbConnectType { + + ORACLE_SERVICE_NAME(0, "Oracle Service Name"), + ORACLE_SID(1, "Oracle SID"); + + DbConnectType(int code, String descp) { + this.code = code; + this.descp = descp; + } + + private final int code; + + private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java new file mode 100644 index 0000000000..0c88407a02 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.enums; + +import static java.util.stream.Collectors.toMap; + +import java.util.Arrays; +import java.util.Map; + +import com.google.common.base.Functions; + +public enum DbType { + + MYSQL(0), + POSTGRESQL(1), + HIVE(2), + SPARK(3), + CLICKHOUSE(4), + ORACLE(5), + SQLSERVER(6), + DB2(7), + PRESTO(8), + H2(9); + + DbType(int code) { + this.code = code; + } + + private final int code; + + public int getCode() { + return code; + } + + private static final Map DB_TYPE_MAP = + Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); + + public static DbType of(int type) { + if (DB_TYPE_MAP.containsKey(type)) { + return DB_TYPE_MAP.get(type); + } + return null; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java new file mode 100644 index 0000000000..0640259e0d --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.enums; + +/** + * have_script + * have_file + * can_retry + * have_arr_variables + * have_map_variables + * have_alert + */ +public enum Flag { + /** + * 0 no + * 1 yes + */ + NO(0, "no"), + YES(1, "yes"); + + Flag(int code, String descp) { + this.code = code; + this.descp = descp; + } + + private final int code; + private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResUploadType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResUploadType.java new file mode 100644 index 0000000000..78bbbed9a8 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResUploadType.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.enums; + +/** + * data base types + */ +public enum ResUploadType { + /** + * 0 hdfs + * 1 s3 + * 2 none + */ + HDFS,S3,NONE +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/TaskTimeoutStrategy.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/TaskTimeoutStrategy.java new file mode 100644 index 0000000000..9df8d55abe --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/TaskTimeoutStrategy.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.enums; + +/** + * task timeout strategy + */ +public enum TaskTimeoutStrategy { + /** + * 0 warn + * 1 failed + * 2 warn+failed + */ + WARN(0, "warn"), + FAILED(1,"failed"), + WARNFAILED(2,"warnfailed"); + + TaskTimeoutStrategy(int code, String descp) { + this.code = code; + this.descp = descp; + } + + private final int code; + private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } + + public static TaskTimeoutStrategy of(int status) { + for (TaskTimeoutStrategy es : values()) { + if (es.getCode() == status) { + return es; + } + } + throw new IllegalArgumentException("invalid status : " + status); + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/AbstractTask.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/AbstractTask.java index 4c354ef1aa..b0ab09aeb2 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/AbstractTask.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/AbstractTask.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.spi.task; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; + /** * executive task */ diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/Property.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/Property.java index edd27589cf..cd6e3c1615 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/Property.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/Property.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.spi.task; -import org.apache.dolphinscheduler.spi.params.base.DataType; +import org.apache.dolphinscheduler.spi.enums.DataType; import java.io.Serializable; import java.util.Objects; diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskChannel.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskChannel.java index 9ffd545291..7ab3b17aeb 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskChannel.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskChannel.java @@ -15,6 +15,8 @@ package org.apache.dolphinscheduler.spi.task;/* * limitations under the License. */ +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; + public interface TaskChannel { void cancelApplication(boolean status); diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskConstants.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskConstants.java index 6c34302e44..b7d5cbb01e 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskConstants.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskConstants.java @@ -41,6 +41,54 @@ public class TaskConstants { */ public static final String COMMA = ","; + /** + * slash / + */ + public static final String SLASH = "/"; + + /** + * COLON : + */ + public static final String COLON = ":"; + + /** + * SPACE " " + */ + public static final String SPACE = " "; + + /** + * SINGLE_SLASH / + */ + public static final String SINGLE_SLASH = "/"; + + /** + * DOUBLE_SLASH // + */ + public static final String DOUBLE_SLASH = "//"; + + /** + * SINGLE_QUOTES "'" + */ + public static final String SINGLE_QUOTES = "'"; + /** + * DOUBLE_QUOTES "\"" + */ + public static final String DOUBLE_QUOTES = "\""; + + /** + * SEMICOLON ; + */ + public static final String SEMICOLON = ";"; + + /** + * EQUAL SIGN + */ + public static final String EQUAL_SIGN = "="; + /** + * AT SIGN + */ + public static final String AT_SIGN = "@"; + /** * sleep time */ @@ -75,4 +123,197 @@ public class TaskConstants { public static final String RWXR_XR_X = "rwxr-xr-x"; + /** + * task log info format + */ + public static final String TASK_LOG_INFO_FORMAT = "TaskLogInfo-%s"; + + /** + * date format of yyyyMMdd + */ + public static final String PARAMETER_FORMAT_DATE = "yyyyMMdd"; + + /** + * date format of yyyyMMddHHmmss + */ + public static final String PARAMETER_FORMAT_TIME = "yyyyMMddHHmmss"; + + /** + * new + * schedule time + */ + public static final String PARAMETER_SHECDULE_TIME = "schedule.time"; + + /** + * system date(yyyyMMddHHmmss) + */ + public static final String PARAMETER_DATETIME = "system.datetime"; + + /** + * system date(yyyymmdd) today + */ + public static final String PARAMETER_CURRENT_DATE = "system.biz.curdate"; + + /** + * system date(yyyymmdd) yesterday + */ + public static final String PARAMETER_BUSINESS_DATE = "system.biz.date"; + + /** + * the absolute path of current executing task + */ + public static final String PARAMETER_TASK_EXECUTE_PATH = "system.task.execute.path"; + + /** + * the instance id of current task + */ + public static final String PARAMETER_TASK_INSTANCE_ID = "system.task.instance.id"; + + /** + * month_begin + */ + public static final String MONTH_BEGIN = "month_begin"; + /** + * add_months + */ + public static final String ADD_MONTHS = "add_months"; + /** + * month_end + */ + public static final String MONTH_END = "month_end"; + /** + * week_begin + */ + public static final String WEEK_BEGIN = "week_begin"; + /** + * week_end + */ + public static final String WEEK_END = "week_end"; + /** + * timestamp + */ + public static final String TIMESTAMP = "timestamp"; + public static final char SUBTRACT_CHAR = '-'; + public static final char ADD_CHAR = '+'; + public static final char MULTIPLY_CHAR = '*'; + public static final char DIVISION_CHAR = '/'; + public static final char LEFT_BRACE_CHAR = '('; + public static final char RIGHT_BRACE_CHAR = ')'; + public static final String ADD_STRING = "+"; + public static final String MULTIPLY_STRING = "*"; + public static final String DIVISION_STRING = "/"; + public static final String LEFT_BRACE_STRING = "("; + public static final char P = 'P'; + public static final char N = 'N'; + public static final String SUBTRACT_STRING = "-"; + public static final String GLOBAL_PARAMS = "globalParams"; + public static final String LOCAL_PARAMS = "localParams"; + public static final String LOCAL_PARAMS_LIST = "localParamsList"; + public static final String SUBPROCESS_INSTANCE_ID = "subProcessInstanceId"; + public static final String PROCESS_INSTANCE_STATE = "processInstanceState"; + public static final String PARENT_WORKFLOW_INSTANCE = "parentWorkflowInstance"; + public static final String CONDITION_RESULT = "conditionResult"; + public static final String SWITCH_RESULT = "switchResult"; + public static final String DEPENDENCE = "dependence"; + public static final String TASK_TYPE = "taskType"; + public static final String TASK_LIST = "taskList"; + public static final String QUEUE = "queue"; + public static final String QUEUE_NAME = "queueName"; + public static final int LOG_QUERY_SKIP_LINE_NUMBER = 0; + public static final int LOG_QUERY_LIMIT = 4096; + + /** + * jar + */ + public static final String JAR = "jar"; + + /** + * hadoop + */ + public static final String HADOOP = "hadoop"; + + /** + * -D = + */ + public static final String D = "-D"; + + /** + * jdbc url + */ + public static final String JDBC_MYSQL = "jdbc:mysql://"; + public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; + public static final String JDBC_HIVE_2 = "jdbc:hive2://"; + public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://"; + public static final String JDBC_ORACLE_SID = "jdbc:oracle:thin:@"; + public static final String JDBC_ORACLE_SERVICE_NAME = "jdbc:oracle:thin:@//"; + public static final String JDBC_SQLSERVER = "jdbc:sqlserver://"; + public static final String JDBC_DB2 = "jdbc:db2://"; + public static final String JDBC_PRESTO = "jdbc:presto://"; + + /** + * driver + */ + public static final String ORG_POSTGRESQL_DRIVER = "org.postgresql.Driver"; + public static final String COM_MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver"; + public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"; + public static final String COM_CLICKHOUSE_JDBC_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; + public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.driver.OracleDriver"; + public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; + public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"; + + /** + * datasource encryption salt + */ + public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*"; + public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable"; + public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt"; + + /** + * resource storage type + */ + public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type"; + + /** + * kerberos + */ + public static final String KERBEROS = "kerberos"; + + /** + * kerberos expire time + */ + public static final String KERBEROS_EXPIRE_TIME = "kerberos.expire.time"; + + /** + * java.security.krb5.conf + */ + public static final String JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + + /** + * java.security.krb5.conf.path + */ + public static final String JAVA_SECURITY_KRB5_CONF_PATH = "java.security.krb5.conf.path"; + + /** + * loginUserFromKeytab user + */ + public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username"; + + /** + * loginUserFromKeytab path + */ + public static final String LOGIN_USER_KEY_TAB_PATH = "login.user.keytab.path"; + + + /** + * hadoop.security.authentication + */ + public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; + + /** + * hadoop.security.authentication + */ + public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state"; + + } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/UdfFuncBean.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/UdfFuncBean.java new file mode 100644 index 0000000000..52e77d60c8 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/UdfFuncBean.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task; + +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.io.IOException; + +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.KeyDeserializer; + +/** + * udf function + */ +public class UdfFuncBean { + /** + * id + */ + private int id; + /** + * user id + */ + private int userId; + + /** + * udf function name + */ + private String funcName; + + /** + * udf class name + */ + private String className; + + /** + * udf argument types + */ + private String argTypes; + + /** + * udf data base + */ + private String database; + + /** + * udf description + */ + private String description; + + /** + * resource id + */ + private int resourceId; + + /** + * resource name + */ + private String resourceName; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getUserId() { + return userId; + } + + public void setUserId(int userId) { + this.userId = userId; + } + + public String getFuncName() { + return funcName; + } + + public void setFuncName(String funcName) { + this.funcName = funcName; + } + + public String getClassName() { + return className; + } + + public void setClassName(String className) { + this.className = className; + } + + public String getArgTypes() { + return argTypes; + } + + public void setArgTypes(String argTypes) { + this.argTypes = argTypes; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public int getResourceId() { + return resourceId; + } + + public void setResourceId(int resourceId) { + this.resourceId = resourceId; + } + + public String getResourceName() { + return resourceName; + } + + public void setResourceName(String resourceName) { + this.resourceName = resourceName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + UdfFuncBean udfFunc = (UdfFuncBean) o; + + if (id != udfFunc.id) { + return false; + } + return !(funcName != null ? !funcName.equals(udfFunc.funcName) : udfFunc.funcName != null); + + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + (funcName != null ? funcName.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return JSONUtils.toJsonString(this); + } + + public static class UdfFuncDeserializer extends KeyDeserializer { + + @Override + public Object deserializeKey(String key, DeserializationContext ctxt) throws IOException { + if (StringUtils.isBlank(key)) { + return null; + } + return JSONUtils.parseObject(key, UdfFuncBean.class); + } + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/AbstractDatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/AbstractDatasourceProcessor.java new file mode 100644 index 0000000000..2e5fa2efd3 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/AbstractDatasourceProcessor.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource; + +import org.apache.commons.collections4.MapUtils; + +import java.util.Map; +import java.util.regex.Pattern; + +public abstract class AbstractDatasourceProcessor implements DatasourceProcessor { + + private static final Pattern IPV4_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); + + private static final Pattern IPV6_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.\\:\\[\\]]+$"); + + private static final Pattern DATABASE_PATTER = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); + + private static final Pattern PARAMS_PATTER = Pattern.compile("^[a-zA-Z0-9\\-\\_\\/]+$"); + + @Override + public void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { + checkHost(baseDataSourceParamDTO.getHost()); + checkDatasourcePatter(baseDataSourceParamDTO.getDatabase()); + checkOther(baseDataSourceParamDTO.getOther()); + } + + /** + * Check the host is valid + * + * @param host datasource host + */ + protected void checkHost(String host) { + if (!IPV4_PATTERN.matcher(host).matches() || !IPV6_PATTERN.matcher(host).matches()) { + throw new IllegalArgumentException("datasource host illegal"); + } + } + + /** + * check database name is valid + * + * @param database database name + */ + protected void checkDatasourcePatter(String database) { + if (!DATABASE_PATTER.matcher(database).matches()) { + throw new IllegalArgumentException("datasource name illegal"); + } + } + + /** + * check other is valid + * + * @param other other + */ + protected void checkOther(Map other) { + if (MapUtils.isEmpty(other)) { + return; + } + boolean paramsCheck = other.entrySet().stream().allMatch(p -> PARAMS_PATTER.matcher(p.getValue()).matches()); + if (!paramsCheck) { + throw new IllegalArgumentException("datasource other params illegal"); + } + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseConnectionParam.java new file mode 100644 index 0000000000..087c2000c8 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseConnectionParam.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource; + +import org.apache.dolphinscheduler.spi.task.datasource.clickhouse.ClickhouseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.db2.Db2ConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.hive.HiveConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.mysql.MysqlConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.oracle.OracleConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.postgresql.PostgreSqlConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.presto.PrestoConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.spark.SparkConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.sqlserver.SqlServerConnectionParam; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; + +/** + * The base model of connection param + *

+ * {@link ClickhouseConnectionParam} + * {@link Db2ConnectionParam} + * {@link HiveConnectionParam} + * {@link MysqlConnectionParam} + * {@link OracleConnectionParam} + * {@link PostgreSqlConnectionParam} + * {@link PrestoConnectionParam} + * {@link SparkConnectionParam} + * {@link SqlServerConnectionParam} + */ +@JsonInclude(Include.NON_NULL) +public abstract class BaseConnectionParam implements ConnectionParam { + + protected String user; + + protected String password; + + protected String address; + + protected String database; + + protected String jdbcUrl; + + protected String other; + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getJdbcUrl() { + return jdbcUrl; + } + + public void setJdbcUrl(String jdbcUrl) { + this.jdbcUrl = jdbcUrl; + } + + public String getOther() { + return other; + } + + public void setOther(String other) { + this.other = other; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseDataSourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseDataSourceParamDTO.java new file mode 100644 index 0000000000..c9ced44ecd --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseDataSourceParamDTO.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.clickhouse.ClickHouseDatasourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.db2.Db2DatasourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.hive.HiveDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.mysql.MysqlDatasourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.oracle.OracleDatasourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.postgresql.PostgreSqlDatasourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.presto.PrestoDatasourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.spark.SparkDatasourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.sqlserver.SqlServerDatasourceParamDTO; + +import java.io.Serializable; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +/** + * Basic datasource params submitted to api. + *

+ * see {@link MysqlDatasourceParamDTO} + * see {@link PostgreSqlDatasourceParamDTO} + * see {@link HiveDataSourceParamDTO} + * see {@link SparkDatasourceParamDTO} + * see {@link ClickHouseDatasourceParamDTO} + * see {@link OracleDatasourceParamDTO} + * see {@link SqlServerDatasourceParamDTO} + * see {@link Db2DatasourceParamDTO} + * see {@link PrestoDatasourceParamDTO} + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(value = MysqlDatasourceParamDTO.class, name = "MYSQL"), + @JsonSubTypes.Type(value = PostgreSqlDatasourceParamDTO.class, name = "POSTGRESQL"), + @JsonSubTypes.Type(value = HiveDataSourceParamDTO.class, name = "HIVE"), + @JsonSubTypes.Type(value = SparkDatasourceParamDTO.class, name = "SPARK"), + @JsonSubTypes.Type(value = ClickHouseDatasourceParamDTO.class, name = "CLICKHOUSE"), + @JsonSubTypes.Type(value = OracleDatasourceParamDTO.class, name = "ORACLE"), + @JsonSubTypes.Type(value = SqlServerDatasourceParamDTO.class, name = "SQLSERVER"), + @JsonSubTypes.Type(value = Db2DatasourceParamDTO.class, name = "DB2"), + @JsonSubTypes.Type(value = PrestoDatasourceParamDTO.class, name = "PRESTO"), +}) +public abstract class BaseDataSourceParamDTO implements Serializable { + + protected Integer id; + + protected String name; + + protected String note; + + protected String host; + + protected Integer port; + + protected String database; + + protected String userName; + + protected String password; + + protected Map other; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public Integer getPort() { + return port; + } + + public void setPort(Integer port) { + this.port = port; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public Map getOther() { + return other; + } + + public void setOther(Map other) { + this.other = other; + } + + /** + * Get the datasource type + * see{@link DbType} + * + * @return datasource type code + */ + public abstract DbType getType(); +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseHdfsConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseHdfsConnectionParam.java new file mode 100644 index 0000000000..e3f8eb9a74 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseHdfsConnectionParam.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource; + +public class BaseHdfsConnectionParam extends BaseConnectionParam { + protected String principal; + protected String javaSecurityKrb5Conf; + protected String loginUserKeytabUsername; + protected String loginUserKeytabPath; + + public String getPrincipal() { + return principal; + } + + public void setPrincipal(String principal) { + this.principal = principal; + } + + public String getJavaSecurityKrb5Conf() { + return javaSecurityKrb5Conf; + } + + public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) { + this.javaSecurityKrb5Conf = javaSecurityKrb5Conf; + } + + public String getLoginUserKeytabUsername() { + return loginUserKeytabUsername; + } + + public void setLoginUserKeytabUsername(String loginUserKeytabUsername) { + this.loginUserKeytabUsername = loginUserKeytabUsername; + } + + public String getLoginUserKeytabPath() { + return loginUserKeytabPath; + } + + public void setLoginUserKeytabPath(String loginUserKeytabPath) { + this.loginUserKeytabPath = loginUserKeytabPath; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseHdfsDatasourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseHdfsDatasourceParamDTO.java new file mode 100644 index 0000000000..5e03ceb394 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/BaseHdfsDatasourceParamDTO.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource; + +public abstract class BaseHdfsDatasourceParamDTO extends BaseDataSourceParamDTO { + + protected String principal; + + protected String javaSecurityKrb5Conf; + + protected String loginUserKeytabUsername; + + protected String loginUserKeytabPath; + + public String getPrincipal() { + return principal; + } + + public void setPrincipal(String principal) { + this.principal = principal; + } + + public String getLoginUserKeytabUsername() { + return loginUserKeytabUsername; + } + + public void setLoginUserKeytabUsername(String loginUserKeytabUsername) { + this.loginUserKeytabUsername = loginUserKeytabUsername; + } + + public String getLoginUserKeytabPath() { + return loginUserKeytabPath; + } + + public void setLoginUserKeytabPath(String loginUserKeytabPath) { + this.loginUserKeytabPath = loginUserKeytabPath; + } + + public String getJavaSecurityKrb5Conf() { + return javaSecurityKrb5Conf; + } + + public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) { + this.javaSecurityKrb5Conf = javaSecurityKrb5Conf; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/ConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/ConnectionParam.java new file mode 100644 index 0000000000..403ebe2841 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/ConnectionParam.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource; + +import java.io.Serializable; + +/** + * The model of Datasource Connection param + */ +public interface ConnectionParam extends Serializable { +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/DatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/DatasourceProcessor.java new file mode 100644 index 0000000000..fe888185d6 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/DatasourceProcessor.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource; + +import org.apache.dolphinscheduler.spi.enums.DbType; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; + +public interface DatasourceProcessor { + + /** + * check datasource param is valid + */ + void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam); + + /** + * create BaseDataSourceParamDTO by connectionJson + * + * @param connectionJson see{@link org.apache.dolphinscheduler.dao.entity.Datasource} + * @return {@link BaseDataSourceParamDTO} + */ + BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson); + + /** + * create datasource connection parameter which will be stored at DataSource + *

+ * see {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams} + */ + ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam); + + /** + * deserialize json to datasource connection param + * + * @param connectionJson {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams} + * @return {@link BaseConnectionParam} + */ + ConnectionParam createConnectionParams(String connectionJson); + + /** + * get datasource Driver + */ + String getDatasourceDriver(); + + /** + * get jdbcUrl by connection param, the jdbcUrl is different with ConnectionParam.jdbcUrl, this method will inject + * other to jdbcUrl + * + * @param connectionParam connection param + */ + String getJdbcUrl(ConnectionParam connectionParam); + + /** + * get connection by connectionParam + * + * @param connectionParam connectionParam + * @return {@link Connection} + */ + Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException, IOException; + + /** + * @return {@link DbType} + */ + DbType getDbType(); +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/DatasourceUtil.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/DatasourceUtil.java new file mode 100644 index 0000000000..c5c4754916 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/DatasourceUtil.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.clickhouse.ClickHouseDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.db2.Db2DatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.hive.HiveDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.mysql.MysqlDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.oracle.OracleDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.postgresql.PostgreSqlDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.presto.PrestoDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.spark.SparkDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.sqlserver.SqlServerDatasourceProcessor; + +import java.sql.Connection; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DatasourceUtil { + + private DatasourceUtil() { + } + + private static final Logger logger = LoggerFactory.getLogger(DatasourceUtil.class); + + private static final DatasourceProcessor mysqlProcessor = new MysqlDatasourceProcessor(); + private static final DatasourceProcessor postgreSqlProcessor = new PostgreSqlDatasourceProcessor(); + private static final DatasourceProcessor hiveProcessor = new HiveDatasourceProcessor(); + private static final DatasourceProcessor sparkProcessor = new SparkDatasourceProcessor(); + private static final DatasourceProcessor clickhouseProcessor = new ClickHouseDatasourceProcessor(); + private static final DatasourceProcessor oracleProcessor = new OracleDatasourceProcessor(); + private static final DatasourceProcessor sqlServerProcessor = new SqlServerDatasourceProcessor(); + private static final DatasourceProcessor db2PROCESSOR = new Db2DatasourceProcessor(); + private static final DatasourceProcessor prestoPROCESSOR = new PrestoDatasourceProcessor(); + + /** + * check datasource param + * + * @param baseDataSourceParamDTO datasource param + */ + public static void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { + getDatasourceProcessor(baseDataSourceParamDTO.getType()).checkDatasourceParam(baseDataSourceParamDTO); + } + + /** + * build connection url + * + * @param baseDataSourceParamDTO datasourceParam + */ + public static ConnectionParam buildConnectionParams(BaseDataSourceParamDTO baseDataSourceParamDTO) { + ConnectionParam connectionParams = getDatasourceProcessor(baseDataSourceParamDTO.getType()) + .createConnectionParams(baseDataSourceParamDTO); + if (logger.isDebugEnabled()) { + logger.info("parameters map:{}", connectionParams); + } + return connectionParams; + } + + public static ConnectionParam buildConnectionParams(DbType dbType, String connectionJson) { + return getDatasourceProcessor(dbType).createConnectionParams(connectionJson); + } + + public static Connection getConnection(DbType dbType, ConnectionParam connectionParam) { + try { + return getDatasourceProcessor(dbType).getConnection(connectionParam); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static String getJdbcUrl(DbType dbType, ConnectionParam baseConnectionParam) { + return getDatasourceProcessor(dbType).getJdbcUrl(baseConnectionParam); + } + + public static BaseDataSourceParamDTO buildDatasourceParamDTO(DbType dbType, String connectionParams) { + return getDatasourceProcessor(dbType).createDatasourceParamDTO(connectionParams); + } + + public static DatasourceProcessor getDatasourceProcessor(DbType dbType) { + switch (dbType) { + case MYSQL: + return mysqlProcessor; + case POSTGRESQL: + return postgreSqlProcessor; + case HIVE: + return hiveProcessor; + case SPARK: + return sparkProcessor; + case CLICKHOUSE: + return clickhouseProcessor; + case ORACLE: + return oracleProcessor; + case SQLSERVER: + return sqlServerProcessor; + case DB2: + return db2PROCESSOR; + case PRESTO: + return prestoPROCESSOR; + default: + throw new IllegalArgumentException("datasource type illegal:" + dbType); + } + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/HiveConfUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/HiveConfUtils.java new file mode 100644 index 0000000000..96485b27cb --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/HiveConfUtils.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * hive conf utils + */ +public class HiveConfUtils { + + private HiveConfUtils() { + throw new UnsupportedOperationException("Construct HiveConfUtils"); + } + + private static class HiveConfHandler { + private static HiveConf singleton; + + private static Map hiveConfVars; + + static { + singleton = new HiveConf(); + hiveConfVars = new HashMap<>(); + Arrays.stream(ConfVars.values()).forEach(confVar -> hiveConfVars.put(confVar.varname,confVar)); + } + } + + /** + * get HiveConf instance + * @return HiveConf hiveConf + */ + public static HiveConf getInstance() { + return HiveConfHandler.singleton; + } + + /** + * get hive conf vars + * @return + */ + public static Map getHiveConfVars() { + return HiveConfHandler.hiveConfVars; + } + + /** + * Determine if it belongs to a hive conf property + * @param conf config + * @return boolean result + */ + public static boolean isHiveConfVar(String conf) { + // the default hive conf var name + String confKey = conf.split("=")[0]; + Map hiveConfVars = HiveConfUtils.getHiveConfVars(); + if (hiveConfVars.get(confKey) != null) { + return true; + } + + // the security authorization hive conf var name + HiveConf hiveConf = HiveConfUtils.getInstance(); + String hiveAuthorizationSqlStdAuthConfigWhitelist = hiveConf.getVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST); + Pattern modWhiteListPattern = Pattern.compile(hiveAuthorizationSqlStdAuthConfigWhitelist); + Matcher matcher = modWhiteListPattern.matcher(confKey); + return matcher.matches(); + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/PasswordUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/PasswordUtils.java new file mode 100644 index 0000000000..a20cd332bd --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/PasswordUtils.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DATASOURCE_ENCRYPTION_ENABLE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DATASOURCE_ENCRYPTION_SALT; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT; + +import org.apache.dolphinscheduler.spi.utils.PropertyUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.codec.binary.Base64; + +import java.nio.charset.StandardCharsets; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PasswordUtils { + + private static final Logger logger = LoggerFactory.getLogger(PasswordUtils.class); + + private static final Base64 BASE64 = new Base64(); + + private PasswordUtils() { + throw new UnsupportedOperationException("Construct PasswordUtils"); + } + + /** + * encode password + */ + public static String encodePassword(String password) { + if (StringUtils.isEmpty(password)) { + return StringUtils.EMPTY; + } + //if encryption is not turned on, return directly + boolean encryptionEnable = PropertyUtils.getBoolean(DATASOURCE_ENCRYPTION_ENABLE, false); + if (!encryptionEnable) { + return password; + } + + // Using Base64 + salt to process password + String salt = PropertyUtils.getString(DATASOURCE_ENCRYPTION_SALT, DATASOURCE_ENCRYPTION_SALT_DEFAULT); + String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes( + StandardCharsets.UTF_8))); + return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8))); + } + + /** + * decode password + */ + public static String decodePassword(String password) { + if (StringUtils.isEmpty(password)) { + return StringUtils.EMPTY; + } + + //if encryption is not turned on, return directly + boolean encryptionEnable = PropertyUtils.getBoolean(DATASOURCE_ENCRYPTION_ENABLE, false); + if (!encryptionEnable) { + return password; + } + + // Using Base64 + salt to process password + String salt = PropertyUtils.getString(DATASOURCE_ENCRYPTION_SALT, DATASOURCE_ENCRYPTION_SALT_DEFAULT); + String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8); + if (!passwordWithSalt.startsWith(salt)) { + logger.warn("There is a password and salt mismatch: {} ", password); + return password; + } + return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8); + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickHouseDatasourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickHouseDatasourceParamDTO.java new file mode 100644 index 0000000000..750552b196 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickHouseDatasourceParamDTO.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.clickhouse; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; + +public class ClickHouseDatasourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "ClickHouseDatasourceParamDTO{" + + "host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.CLICKHOUSE; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java new file mode 100644 index 0000000000..fcc3bc7f56 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickHouseDatasourceProcessor.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.clickhouse; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_CLICKHOUSE_JDBC_DRIVER; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_CLICKHOUSE; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.encodePassword; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) createConnectionParams(connectionJson); + + ClickHouseDatasourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDatasourceParamDTO(); + clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser()); + clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); + clickHouseDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); + clickHouseDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); + + return clickHouseDatasourceParamDTO; + } + + @Override + public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + ClickHouseDatasourceParamDTO clickHouseParam = (ClickHouseDatasourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", JDBC_CLICKHOUSE, clickHouseParam.getHost(), clickHouseParam.getPort()); + String jdbcUrl = address + "/" + clickHouseParam.getDatabase(); + + ClickhouseConnectionParam clickhouseConnectionParam = new ClickhouseConnectionParam(); + clickhouseConnectionParam.setDatabase(clickHouseParam.getDatabase()); + clickhouseConnectionParam.setAddress(address); + clickhouseConnectionParam.setJdbcUrl(jdbcUrl); + clickhouseConnectionParam.setUser(clickHouseParam.getUserName()); + clickhouseConnectionParam.setPassword(encodePassword(clickHouseParam.getPassword())); + clickhouseConnectionParam.setOther(transformOther(clickHouseParam.getOther())); + return clickhouseConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, ClickhouseConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return COM_CLICKHOUSE_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; + String jdbcUrl = clickhouseConnectionParam.getJdbcUrl(); + if (StringUtils.isNotEmpty(clickhouseConnectionParam.getOther())) { + jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther()); + } + return jdbcUrl; + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam), + clickhouseConnectionParam.getUser(), decodePassword(clickhouseConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.CLICKHOUSE; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, "&"))); + return stringBuilder.toString(); + } + + private Map parseOther(String other) { + if (other == null) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + String[] configs = other.split("&"); + for (String config : configs) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickhouseConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickhouseConnectionParam.java new file mode 100644 index 0000000000..38f51f4efb --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/clickhouse/ClickhouseConnectionParam.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.clickhouse; + +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; + +public class ClickhouseConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "ClickhouseConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2ConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2ConnectionParam.java new file mode 100644 index 0000000000..1460ed7326 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2ConnectionParam.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.db2; + +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; + +public class Db2ConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "Db2ConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2DatasourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2DatasourceParamDTO.java new file mode 100644 index 0000000000..d186e75820 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2DatasourceParamDTO.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.db2; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; + +public class Db2DatasourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "Db2DatasourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.DB2; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2DatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2DatasourceProcessor.java new file mode 100644 index 0000000000..2452870f26 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/db2/Db2DatasourceProcessor.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.db2; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_DB2_JDBC_DRIVER; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_DB2; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.encodePassword; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +public class Db2DatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + Db2ConnectionParam connectionParams = (Db2ConnectionParam) createConnectionParams(connectionJson); + + Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO(); + db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName()); + + String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); + db2DatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); + db2DatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); + + return db2DatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + Db2DatasourceParamDTO db2Param = (Db2DatasourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", JDBC_DB2, db2Param.getHost(), db2Param.getPort()); + String jdbcUrl = String.format("%s/%s", address, db2Param.getDatabase()); + + Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam(); + db2ConnectionParam.setAddress(address); + db2ConnectionParam.setDatabase(db2Param.getDatabase()); + db2ConnectionParam.setJdbcUrl(jdbcUrl); + db2ConnectionParam.setUser(db2Param.getUserName()); + db2ConnectionParam.setPassword(encodePassword(db2Param.getPassword())); + db2ConnectionParam.setOther(transformOther(db2Param.getOther())); + + return db2ConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, Db2ConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return COM_DB2_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; + if (StringUtils.isNotEmpty(db2ConnectionParam.getOther())) { + return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther()); + } + return db2ConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(db2ConnectionParam), + db2ConnectionParam.getUser(), decodePassword(db2ConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.DB2; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, ";"))); + stringBuilder.deleteCharAt(stringBuilder.length() - 1); + return stringBuilder.toString(); + } + + private Map parseOther(String other) { + if (other == null) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + for (String config : other.split("&")) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveConnectionParam.java new file mode 100644 index 0000000000..e77d362822 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveConnectionParam.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.hive; + +import org.apache.dolphinscheduler.spi.task.datasource.BaseHdfsConnectionParam; + +public class HiveConnectionParam extends BaseHdfsConnectionParam { + @Override + public String toString() { + return "HiveConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + ", principal='" + principal + '\'' + + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' + + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' + + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveDataSourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveDataSourceParamDTO.java new file mode 100644 index 0000000000..db967888b7 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveDataSourceParamDTO.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.hive; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseHdfsDatasourceParamDTO; + +public class HiveDataSourceParamDTO extends BaseHdfsDatasourceParamDTO { + + @Override + public String toString() { + return "HiveDataSourceParamDTO{" + + "host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", principal='" + principal + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' + + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' + + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.HIVE; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveDatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveDatasourceProcessor.java new file mode 100644 index 0000000000..e4360e496d --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/hive/HiveDatasourceProcessor.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.hive; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_HIVE_2; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.encodePassword; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.HiveConfUtils; +import org.apache.dolphinscheduler.spi.utils.CommonUtils; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); + HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) createConnectionParams(connectionJson); + + hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase()); + hiveDataSourceParamDTO.setUserName(hiveConnectionParam.getUser()); + hiveDataSourceParamDTO.setOther(parseOther(hiveConnectionParam.getOther())); + hiveDataSourceParamDTO.setLoginUserKeytabUsername(hiveConnectionParam.getLoginUserKeytabUsername()); + hiveDataSourceParamDTO.setLoginUserKeytabPath(hiveConnectionParam.getLoginUserKeytabPath()); + hiveDataSourceParamDTO.setJavaSecurityKrb5Conf(hiveConnectionParam.getJavaSecurityKrb5Conf()); + + String[] tmpArray = hiveConnectionParam.getAddress().split(DOUBLE_SLASH); + StringBuilder hosts = new StringBuilder(); + String[] hostPortArray = tmpArray[tmpArray.length - 1].split(COMMA); + for (String hostPort : hostPortArray) { + hosts.append(hostPort.split(COLON)[0]).append(COMMA); + } + hosts.deleteCharAt(hosts.length() - 1); + hiveDataSourceParamDTO.setHost(hosts.toString()); + hiveDataSourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); + + return hiveDataSourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + HiveDataSourceParamDTO hiveParam = (HiveDataSourceParamDTO) datasourceParam; + StringBuilder address = new StringBuilder(); + address.append(JDBC_HIVE_2); + for (String zkHost : hiveParam.getHost().split(",")) { + address.append(String.format("%s:%s,", zkHost, hiveParam.getPort())); + } + address.deleteCharAt(address.length() - 1); + String jdbcUrl = address.toString() + "/" + hiveParam.getDatabase(); + if (CommonUtils.getKerberosStartupState()) { + jdbcUrl += ";principal=" + hiveParam.getPrincipal(); + } + + HiveConnectionParam hiveConnectionParam = new HiveConnectionParam(); + hiveConnectionParam.setDatabase(hiveParam.getDatabase()); + hiveConnectionParam.setAddress(address.toString()); + hiveConnectionParam.setJdbcUrl(jdbcUrl); + hiveConnectionParam.setUser(hiveParam.getUserName()); + hiveConnectionParam.setPassword(encodePassword(hiveParam.getPassword())); + + if (CommonUtils.getKerberosStartupState()) { + hiveConnectionParam.setPrincipal(hiveParam.getPrincipal()); + hiveConnectionParam.setJavaSecurityKrb5Conf(hiveParam.getJavaSecurityKrb5Conf()); + hiveConnectionParam.setLoginUserKeytabPath(hiveParam.getLoginUserKeytabPath()); + hiveConnectionParam.setLoginUserKeytabUsername(hiveParam.getLoginUserKeytabUsername()); + } + hiveConnectionParam.setOther(transformOther(hiveParam.getOther())); + return hiveConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, HiveConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; + String jdbcUrl = hiveConnectionParam.getJdbcUrl(); + String otherParams = filterOther(hiveConnectionParam.getOther()); + if (StringUtils.isNotEmpty(otherParams) && !"?".equals(otherParams.substring(0, 1))) { + jdbcUrl += ";"; + } + return jdbcUrl + otherParams; + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { + HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; + CommonUtils.loadKerberosConf(hiveConnectionParam.getJavaSecurityKrb5Conf(), + hiveConnectionParam.getLoginUserKeytabUsername(), hiveConnectionParam.getLoginUserKeytabPath()); + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(connectionParam), + hiveConnectionParam.getUser(), decodePassword(hiveConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.HIVE; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); + return stringBuilder.toString(); + } + + private String filterOther(String otherParams) { + if (StringUtils.isBlank(otherParams)) { + return ""; + } + + StringBuilder hiveConfListSb = new StringBuilder(); + hiveConfListSb.append("?"); + StringBuilder sessionVarListSb = new StringBuilder(); + + String[] otherArray = otherParams.split(";", -1); + + for (String conf : otherArray) { + if (HiveConfUtils.isHiveConfVar(conf)) { + hiveConfListSb.append(conf).append(";"); + } else { + sessionVarListSb.append(conf).append(";"); + } + } + + // remove the last ";" + if (sessionVarListSb.length() > 0) { + sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); + } + + if (hiveConfListSb.length() > 0) { + hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); + } + + return sessionVarListSb.toString() + hiveConfListSb.toString(); + } + + private Map parseOther(String other) { + if (other == null) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + String[] configs = other.split(";"); + for (String config : configs) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlConnectionParam.java new file mode 100644 index 0000000000..8f3e65bc9f --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlConnectionParam.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.mysql; + +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; + +public class MysqlConnectionParam extends BaseConnectionParam { + + @Override + public String toString() { + return "MysqlConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlDatasourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlDatasourceParamDTO.java new file mode 100644 index 0000000000..48d7aa98d5 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlDatasourceParamDTO.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.mysql; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; + +public class MysqlDatasourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "MysqlDatasourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.MYSQL; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlDatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlDatasourceProcessor.java new file mode 100644 index 0000000000..0ba9671451 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/mysql/MysqlDatasourceProcessor.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.mysql; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_MYSQL_JDBC_DRIVER; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_MYSQL; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.encodePassword; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor { + + private final Logger logger = LoggerFactory.getLogger(MysqlDatasourceProcessor.class); + + private static final String ALLOW_LOAD_LOCAL_IN_FILE_NAME = "allowLoadLocalInfile"; + + private static final String AUTO_DESERIALIZE = "autoDeserialize"; + + private static final String ALLOW_LOCAL_IN_FILE_NAME = "allowLocalInfile"; + + private static final String ALLOW_URL_IN_LOCAL_IN_FILE_NAME = "allowUrlInLocalInfile"; + + private static final String APPEND_PARAMS = "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + MysqlConnectionParam connectionParams = (MysqlConnectionParam) createConnectionParams(connectionJson); + MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); + + mysqlDatasourceParamDTO.setUserName(connectionParams.getUser()); + mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + mysqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + String address = connectionParams.getAddress(); + String[] hostSeperator = address.split(DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); + mysqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); + mysqlDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); + + return mysqlDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { + MysqlDatasourceParamDTO mysqlDatasourceParam = (MysqlDatasourceParamDTO) dataSourceParam; + String address = String.format("%s%s:%s", JDBC_MYSQL, mysqlDatasourceParam.getHost(), mysqlDatasourceParam.getPort()); + String jdbcUrl = String.format("%s/%s", address, mysqlDatasourceParam.getDatabase()); + + MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); + mysqlConnectionParam.setJdbcUrl(jdbcUrl); + mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase()); + mysqlConnectionParam.setAddress(address); + mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName()); + mysqlConnectionParam.setPassword(encodePassword(mysqlDatasourceParam.getPassword())); + mysqlConnectionParam.setOther(transformOther(mysqlDatasourceParam.getOther())); + + return mysqlConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, MysqlConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return COM_MYSQL_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; + String jdbcUrl = mysqlConnectionParam.getJdbcUrl(); + if (StringUtils.isNotEmpty(mysqlConnectionParam.getOther())) { + return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS); + } + return String.format("%s?%s", jdbcUrl, APPEND_PARAMS); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + String user = mysqlConnectionParam.getUser(); + if (user.contains(AUTO_DESERIALIZE)) { + logger.warn("sensitive param : {} in username field is filtered", AUTO_DESERIALIZE); + user = user.replace(AUTO_DESERIALIZE, ""); + } + String password = decodePassword(mysqlConnectionParam.getPassword()); + if (password.contains(AUTO_DESERIALIZE)) { + logger.warn("sensitive param : {} in password field is filtered", AUTO_DESERIALIZE); + password = password.replace(AUTO_DESERIALIZE, ""); + } + return DriverManager.getConnection(getJdbcUrl(connectionParam), user, password); + } + + @Override + public DbType getDbType() { + return DbType.MYSQL; + } + + private String transformOther(Map paramMap) { + if (MapUtils.isEmpty(paramMap)) { + return null; + } + Map otherMap = new HashMap<>(); + paramMap.forEach((k, v) -> { + if (!checkKeyIsLegitimate(k)) { + return; + } + otherMap.put(k, v); + }); + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); + return stringBuilder.toString(); + } + + private static boolean checkKeyIsLegitimate(String key) { + return !key.contains(ALLOW_LOAD_LOCAL_IN_FILE_NAME) + && !key.contains(AUTO_DESERIALIZE) + && !key.contains(ALLOW_LOCAL_IN_FILE_NAME) + && !key.contains(ALLOW_URL_IN_LOCAL_IN_FILE_NAME); + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + for (String config : other.split("&")) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleConnectionParam.java new file mode 100644 index 0000000000..03bf74ddf0 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleConnectionParam.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.oracle; + +import org.apache.dolphinscheduler.spi.enums.DbConnectType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; + +public class OracleConnectionParam extends BaseConnectionParam { + + protected DbConnectType connectType; + + public DbConnectType getConnectType() { + return connectType; + } + + public void setConnectType(DbConnectType connectType) { + this.connectType = connectType; + } + + @Override + public String toString() { + return "OracleConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + ", connectType=" + connectType + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleDatasourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleDatasourceParamDTO.java new file mode 100644 index 0000000000..07cb8da7d7 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleDatasourceParamDTO.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.oracle; + +import org.apache.dolphinscheduler.spi.enums.DbConnectType; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; + +public class OracleDatasourceParamDTO extends BaseDataSourceParamDTO { + + private DbConnectType connectType; + + public DbConnectType getConnectType() { + return connectType; + } + + public void setConnectType(DbConnectType connectType) { + this.connectType = connectType; + } + + @Override + public String toString() { + return "OracleDatasourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", connectType=" + connectType + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.ORACLE; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleDatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleDatasourceProcessor.java new file mode 100644 index 0000000000..7c456a8f16 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/oracle/OracleDatasourceProcessor.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.oracle; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.AT_SIGN; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_ORACLE_JDBC_DRIVER; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_ORACLE_SERVICE_NAME; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_ORACLE_SID; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.encodePassword; + +import org.apache.dolphinscheduler.spi.enums.DbConnectType; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class OracleDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + OracleConnectionParam connectionParams = (OracleConnectionParam) createConnectionParams(connectionJson); + OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO(); + + oracleDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + oracleDatasourceParamDTO.setUserName(connectionParams.getUser()); + oracleDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + String hostSeperator = DOUBLE_SLASH; + if (DbConnectType.ORACLE_SID.equals(connectionParams.connectType)) { + hostSeperator = AT_SIGN; + } + String[] hostPort = connectionParams.getAddress().split(hostSeperator); + String[] hostPortArray = hostPort[hostPort.length - 1].split(COMMA); + oracleDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); + oracleDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); + + return oracleDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + OracleDatasourceParamDTO oracleParam = (OracleDatasourceParamDTO) datasourceParam; + String address; + if (DbConnectType.ORACLE_SID.equals(oracleParam.getConnectType())) { + address = String.format("%s%s:%s", + JDBC_ORACLE_SID, oracleParam.getHost(), oracleParam.getPort()); + } else { + address = String.format("%s%s:%s", + JDBC_ORACLE_SERVICE_NAME, oracleParam.getHost(), oracleParam.getPort()); + } + String jdbcUrl = address + "/" + oracleParam.getDatabase(); + + OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); + oracleConnectionParam.setUser(oracleParam.getUserName()); + oracleConnectionParam.setPassword(encodePassword(oracleParam.getPassword())); + oracleConnectionParam.setAddress(address); + oracleConnectionParam.setJdbcUrl(jdbcUrl); + oracleConnectionParam.setDatabase(oracleParam.getDatabase()); + oracleConnectionParam.setConnectType(oracleParam.getConnectType()); + oracleConnectionParam.setOther(transformOther(oracleParam.getOther())); + + return oracleConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, OracleConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return COM_ORACLE_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; + if (StringUtils.isNotEmpty(oracleConnectionParam.getOther())) { + return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther()); + } + return oracleConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(connectionParam), + oracleConnectionParam.getUser(), decodePassword(oracleConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.ORACLE; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + List list = new ArrayList<>(); + otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); + return String.join("&", list); + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + String[] configs = other.split("&"); + for (String config : configs) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlConnectionParam.java new file mode 100644 index 0000000000..c63096d333 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlConnectionParam.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.postgresql; + +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; + +public class PostgreSqlConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "PostgreSqlConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlDatasourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlDatasourceParamDTO.java new file mode 100644 index 0000000000..14c4aaa636 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlDatasourceParamDTO.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.postgresql; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; + +public class PostgreSqlDatasourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "PostgreSqlDatasourceParamDTO{" + + "host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.POSTGRESQL; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java new file mode 100644 index 0000000000..337809eaaa --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/postgresql/PostgreSqlDatasourceProcessor.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.postgresql; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_POSTGRESQL; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_POSTGRESQL_DRIVER; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.encodePassword; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) createConnectionParams(connectionJson); + PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO(); + postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser()); + postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + String address = connectionParams.getAddress(); + String[] hostSeperator = address.split(DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); + postgreSqlDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); + postgreSqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); + + return postgreSqlDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + PostgreSqlDatasourceParamDTO postgreSqlParam = (PostgreSqlDatasourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", JDBC_POSTGRESQL, postgreSqlParam.getHost(), postgreSqlParam.getPort()); + String jdbcUrl = String.format("%s/%s", address, postgreSqlParam.getDatabase()); + + PostgreSqlConnectionParam postgreSqlConnectionParam = new PostgreSqlConnectionParam(); + postgreSqlConnectionParam.setJdbcUrl(jdbcUrl); + postgreSqlConnectionParam.setAddress(address); + postgreSqlConnectionParam.setDatabase(postgreSqlParam.getDatabase()); + postgreSqlConnectionParam.setUser(postgreSqlParam.getUserName()); + postgreSqlConnectionParam.setPassword(encodePassword(postgreSqlParam.getPassword())); + postgreSqlConnectionParam.setOther(transformOther(postgreSqlParam.getOther())); + + return postgreSqlConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, PostgreSqlConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return ORG_POSTGRESQL_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; + if (StringUtils.isNotEmpty(postgreSqlConnectionParam.getOther())) { + return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther()); + } + return postgreSqlConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam), + postgreSqlConnectionParam.getUser(), decodePassword(postgreSqlConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.POSTGRESQL; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); + return stringBuilder.toString(); + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + for (String config : other.split("&")) { + String[] split = config.split("="); + otherMap.put(split[0], split[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoConnectionParam.java new file mode 100644 index 0000000000..ac142b6ff0 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoConnectionParam.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.presto; + +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; + +public class PrestoConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "PrestoConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoDatasourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoDatasourceParamDTO.java new file mode 100644 index 0000000000..689d6738f8 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoDatasourceParamDTO.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.presto; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; + +public class PrestoDatasourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "PrestoDatasourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.PRESTO; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoDatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoDatasourceProcessor.java new file mode 100644 index 0000000000..55eef138c9 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/presto/PrestoDatasourceProcessor.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.presto; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_PRESTO_JDBC_DRIVER; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_PRESTO; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.encodePassword; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + PrestoConnectionParam connectionParams = (PrestoConnectionParam) createConnectionParams(connectionJson); + + String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); + + PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO(); + prestoDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); + prestoDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); + prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + prestoDatasourceParamDTO.setUserName(connectionParams.getUser()); + prestoDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + return prestoDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + PrestoDatasourceParamDTO prestoParam = (PrestoDatasourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", JDBC_PRESTO, prestoParam.getHost(), prestoParam.getPort()); + String jdbcUrl = address + "/" + prestoParam.getDatabase(); + + PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam(); + prestoConnectionParam.setUser(prestoParam.getUserName()); + prestoConnectionParam.setPassword(encodePassword(prestoParam.getPassword())); + prestoConnectionParam.setOther(transformOther(prestoParam.getOther())); + prestoConnectionParam.setAddress(address); + prestoConnectionParam.setJdbcUrl(jdbcUrl); + prestoConnectionParam.setDatabase(prestoParam.getDatabase()); + + return prestoConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, PrestoConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return COM_PRESTO_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; + if (StringUtils.isNotEmpty(prestoConnectionParam.getOther())) { + return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther()); + } + return prestoConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(connectionParam), + prestoConnectionParam.getUser(), decodePassword(prestoConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.PRESTO; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isNotEmpty(otherMap)) { + List list = new ArrayList<>(); + otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); + return String.join("&", list); + } + return null; + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + String[] configs = other.split("&"); + for (String config : configs) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkConnectionParam.java new file mode 100644 index 0000000000..ff226a20e4 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkConnectionParam.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.spark; + +import org.apache.dolphinscheduler.spi.task.datasource.BaseHdfsConnectionParam; + +public class SparkConnectionParam extends BaseHdfsConnectionParam { + @Override + public String toString() { + return "SparkConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + ", principal='" + principal + '\'' + + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' + + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' + + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkDatasourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkDatasourceParamDTO.java new file mode 100644 index 0000000000..afe204751d --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkDatasourceParamDTO.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.spark; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseHdfsDatasourceParamDTO; + +public class SparkDatasourceParamDTO extends BaseHdfsDatasourceParamDTO { + + @Override + public String toString() { + return "SparkDatasourceParamDTO{" + + "host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", principal='" + principal + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' + + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' + + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.SPARK; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkDatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkDatasourceProcessor.java new file mode 100644 index 0000000000..53b4fd9770 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/spark/SparkDatasourceProcessor.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.spark; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_HIVE_2; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.encodePassword; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.utils.CommonUtils; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class SparkDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + SparkConnectionParam connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson); + + SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO(); + sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + sparkDatasourceParamDTO.setUserName(connectionParams.getUser()); + sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + sparkDatasourceParamDTO.setJavaSecurityKrb5Conf(connectionParams.getJavaSecurityKrb5Conf()); + sparkDatasourceParamDTO.setLoginUserKeytabPath(connectionParams.getLoginUserKeytabPath()); + sparkDatasourceParamDTO.setLoginUserKeytabUsername(connectionParams.getLoginUserKeytabUsername()); + + StringBuilder hosts = new StringBuilder(); + String[] tmpArray = connectionParams.getAddress().split(DOUBLE_SLASH); + String[] hostPortArray = tmpArray[tmpArray.length - 1].split(COMMA); + Arrays.stream(hostPortArray).forEach(hostPort -> hosts.append(hostPort.split(COLON)[0]).append(COMMA)); + hosts.deleteCharAt(hosts.length() - 1); + + sparkDatasourceParamDTO.setHost(hosts.toString()); + sparkDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); + + return sparkDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { + StringBuilder address = new StringBuilder(); + SparkDatasourceParamDTO sparkDatasourceParam = (SparkDatasourceParamDTO) dataSourceParam; + address.append(JDBC_HIVE_2); + for (String zkHost : sparkDatasourceParam.getHost().split(",")) { + address.append(String.format("%s:%s,", zkHost, sparkDatasourceParam.getPort())); + } + address.deleteCharAt(address.length() - 1); + + String jdbcUrl = address + "/" + sparkDatasourceParam.getDatabase(); + if (CommonUtils.getKerberosStartupState()) { + jdbcUrl += ";principal=" + sparkDatasourceParam.getPrincipal(); + } + + SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); + sparkConnectionParam.setPassword(encodePassword(sparkDatasourceParam.getPassword())); + sparkConnectionParam.setUser(sparkDatasourceParam.getUserName()); + sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther())); + sparkConnectionParam.setDatabase(sparkDatasourceParam.getDatabase()); + sparkConnectionParam.setAddress(address.toString()); + sparkConnectionParam.setJdbcUrl(jdbcUrl); + if (CommonUtils.getKerberosStartupState()) { + sparkConnectionParam.setPrincipal(sparkDatasourceParam.getPrincipal()); + sparkConnectionParam.setJavaSecurityKrb5Conf(sparkDatasourceParam.getJavaSecurityKrb5Conf()); + sparkConnectionParam.setLoginUserKeytabPath(sparkDatasourceParam.getLoginUserKeytabPath()); + sparkConnectionParam.setLoginUserKeytabUsername(sparkDatasourceParam.getLoginUserKeytabUsername()); + } + + return sparkConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, SparkConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; + if (StringUtils.isNotEmpty(sparkConnectionParam.getOther())) { + return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther()); + } + return sparkConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { + SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; + CommonUtils.loadKerberosConf(sparkConnectionParam.getJavaSecurityKrb5Conf(), + sparkConnectionParam.getLoginUserKeytabUsername(), sparkConnectionParam.getLoginUserKeytabPath()); + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(sparkConnectionParam), + sparkConnectionParam.getUser(), decodePassword(sparkConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.SPARK; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + List stringBuilder = otherMap.entrySet().stream() + .map(entry -> String.format("%s=%s", entry.getKey(), entry.getValue())).collect(Collectors.toList()); + return String.join(";", stringBuilder); + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + String[] configs = other.split(";"); + for (String config : configs) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerConnectionParam.java new file mode 100644 index 0000000000..8091cd86b3 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerConnectionParam.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.sqlserver; + +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; + +public class SqlServerConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "SqlServerConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerDatasourceParamDTO.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerDatasourceParamDTO.java new file mode 100644 index 0000000000..c6024c8a8d --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerDatasourceParamDTO.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.sqlserver; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; + +public class SqlServerDatasourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "SqlServerDatasourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.SQLSERVER; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerDatasourceProcessor.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerDatasourceProcessor.java new file mode 100644 index 0000000000..6b76a6b0a2 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/datasource/sqlserver/SqlServerDatasourceProcessor.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.datasource.sqlserver; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COLON; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COM_SQLSERVER_JDBC_DRIVER; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_SLASH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JDBC_SQLSERVER; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.encodePassword; + +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.task.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + SqlServerConnectionParam connectionParams = (SqlServerConnectionParam) createConnectionParams(connectionJson); + String[] hostSeperator = connectionParams.getAddress().split(DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(COMMA); + + SqlServerDatasourceParamDTO sqlServerDatasourceParamDTO = new SqlServerDatasourceParamDTO(); + sqlServerDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + sqlServerDatasourceParamDTO.setUserName(connectionParams.getUser()); + sqlServerDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + sqlServerDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(COLON)[1])); + sqlServerDatasourceParamDTO.setHost(hostPortArray[0].split(COLON)[0]); + return sqlServerDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + SqlServerDatasourceParamDTO sqlServerParam = (SqlServerDatasourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort()); + String jdbcUrl = address + ";databaseName=" + sqlServerParam.getDatabase(); + + SqlServerConnectionParam sqlServerConnectionParam = new SqlServerConnectionParam(); + sqlServerConnectionParam.setAddress(address); + sqlServerConnectionParam.setDatabase(sqlServerParam.getDatabase()); + sqlServerConnectionParam.setJdbcUrl(jdbcUrl); + sqlServerConnectionParam.setOther(transformOther(sqlServerParam.getOther())); + sqlServerConnectionParam.setUser(sqlServerParam.getUserName()); + sqlServerConnectionParam.setPassword(encodePassword(sqlServerParam.getPassword())); + return sqlServerConnectionParam; + } + + @Override + public BaseConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, SqlServerConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return COM_SQLSERVER_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; + + if (StringUtils.isNotEmpty(sqlServerConnectionParam.getOther())) { + return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther()); + } + return sqlServerConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(connectionParam), sqlServerConnectionParam.getUser(), + decodePassword(sqlServerConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.SQLSERVER; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); + return stringBuilder.toString(); + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + for (String config : other.split(";")) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/BusinessTimeUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/BusinessTimeUtils.java new file mode 100644 index 0000000000..b35598d42e --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/BusinessTimeUtils.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.paramparser; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_BUSINESS_DATE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_CURRENT_DATE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_DATETIME; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_FORMAT_DATE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_FORMAT_TIME; +import static org.apache.dolphinscheduler.spi.utils.DateUtils.addDays; +import static org.apache.dolphinscheduler.spi.utils.DateUtils.format; + +import org.apache.dolphinscheduler.spi.enums.CommandType; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * business time utils + */ +public class BusinessTimeUtils { + private BusinessTimeUtils() { + throw new IllegalStateException("BusinessTimeUtils class"); + } + + /** + * get business time in parameters by different command types + * + * @param commandType command type + * @param runTime run time or schedule time + * @return business time + */ + public static Map getBusinessTime(CommandType commandType, Date runTime) { + Date businessDate = runTime; + switch (commandType) { + case COMPLEMENT_DATA: + break; + case START_PROCESS: + case START_CURRENT_TASK_PROCESS: + case RECOVER_TOLERANCE_FAULT_PROCESS: + case RECOVER_SUSPENDED_PROCESS: + case START_FAILURE_TASK_PROCESS: + case REPEAT_RUNNING: + case SCHEDULER: + default: + businessDate = addDays(new Date(), -1); + if (runTime != null) { + /** + * If there is a scheduled time, take the scheduling time. Recovery from failed nodes, suspension of recovery, re-run for scheduling + */ + businessDate = addDays(runTime, -1); + } + break; + } + Date businessCurrentDate = addDays(businessDate, 1); + Map result = new HashMap<>(); + result.put(PARAMETER_CURRENT_DATE, format(businessCurrentDate, PARAMETER_FORMAT_DATE)); + result.put(PARAMETER_BUSINESS_DATE, format(businessDate, PARAMETER_FORMAT_DATE)); + result.put(PARAMETER_DATETIME, format(businessCurrentDate, PARAMETER_FORMAT_TIME)); + return result; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/ParamUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/ParamUtils.java new file mode 100644 index 0000000000..069f941ff8 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/ParamUtils.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.paramparser; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_TASK_EXECUTE_PATH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_TASK_INSTANCE_ID; + +import org.apache.dolphinscheduler.spi.enums.CommandType; +import org.apache.dolphinscheduler.spi.enums.DataType; +import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.Direct; +import org.apache.dolphinscheduler.spi.task.Property; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import com.google.common.base.Preconditions; + +/** + * param utils + */ +public class ParamUtils { + + /** + * parameter conversion + * Warning: + * When you first invoke the function of convert, the variables of localParams and varPool in the ShellParameters will be modified. + * But in the whole system the variables of localParams and varPool have been used in other functions. I'm not sure if this current + * situation is wrong. So I cannot modify the original logic. + * + * @param taskExecutionContext the context of this task instance + * @param parameters the parameters + * @return global params + * + */ + public static Map convert(TaskRequest taskExecutionContext, AbstractParameters parameters) { + Preconditions.checkNotNull(taskExecutionContext); + Preconditions.checkNotNull(parameters); + Map globalParams = getUserDefParamsMap(taskExecutionContext.getDefinedParams()); + Map globalParamsMap = taskExecutionContext.getDefinedParams(); + CommandType commandType = CommandType.of(taskExecutionContext.getCmdTypeIfComplement()); + Date scheduleTime = taskExecutionContext.getScheduleTime(); + + // combining local and global parameters + Map localParams = parameters.getLocalParametersMap(); + + Map varParams = parameters.getVarPoolMap(); + + if (globalParams == null && localParams == null) { + return null; + } + // if it is a complement, + // you need to pass in the task instance id to locate the time + // of the process instance complement + Map params = BusinessTimeUtils + .getBusinessTime(commandType, + scheduleTime); + + if (globalParamsMap != null) { + + params.putAll(globalParamsMap); + } + + if (StringUtils.isNotBlank(taskExecutionContext.getExecutePath())) { + params.put(PARAMETER_TASK_EXECUTE_PATH, taskExecutionContext.getExecutePath()); + } + params.put(PARAMETER_TASK_INSTANCE_ID, Integer.toString(taskExecutionContext.getTaskInstanceId())); + + if (globalParams != null && localParams != null) { + globalParams.putAll(localParams); + } else if (globalParams == null && localParams != null) { + globalParams = localParams; + } + if (varParams != null) { + varParams.putAll(globalParams); + globalParams = varParams; + } + Iterator> iter = globalParams.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry en = iter.next(); + Property property = en.getValue(); + + if (StringUtils.isNotEmpty(property.getValue()) + && property.getValue().startsWith("$")) { + /** + * local parameter refers to global parameter with the same name + * note: the global parameters of the process instance here are solidified parameters, + * and there are no variables in them. + */ + String val = property.getValue(); + + val = ParameterUtils.convertParameterPlaceholders(val, params); + property.setValue(val); + } + } + + return globalParams; + } + + /** + * format convert + * + * @param paramsMap params map + * @return Map of converted + */ + public static Map convert(Map paramsMap) { + if (paramsMap == null) { + return null; + } + + Map map = new HashMap<>(); + Iterator> iter = paramsMap.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry en = iter.next(); + map.put(en.getKey(), en.getValue().getValue()); + } + return map; + } + + /** + * get parameters map + * + * @param definedParams definedParams + * @return parameters map + */ + public static Map getUserDefParamsMap(Map definedParams) { + if (definedParams != null) { + Map userDefParamsMaps = new HashMap<>(); + Iterator> iter = definedParams.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry en = iter.next(); + Property property = new Property(en.getKey(), Direct.IN, DataType.VARCHAR, en.getValue()); + userDefParamsMaps.put(property.getProp(),property); + } + return userDefParamsMaps; + } + return null; + } +} \ No newline at end of file diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/ParameterUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/ParameterUtils.java new file mode 100644 index 0000000000..208022f5ca --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/ParameterUtils.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.paramparser; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_DATETIME; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_FORMAT_TIME; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_SHECDULE_TIME; + +import org.apache.dolphinscheduler.spi.enums.CommandType; +import org.apache.dolphinscheduler.spi.enums.DataType; +import org.apache.dolphinscheduler.spi.task.Property; +import org.apache.dolphinscheduler.spi.utils.DateUtils; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.sql.PreparedStatement; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * parameter parse utils + */ +public class ParameterUtils { + + private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class); + + private static final String DATE_PARSE_PATTERN = "\\$\\[([^\\$\\]]+)]"; + + private static final String DATE_START_PATTERN = "^[0-9]"; + + private ParameterUtils() { + throw new UnsupportedOperationException("Construct ParameterUtils"); + } + + /** + * convert parameters place holders + * + * @param parameterString parameter + * @param parameterMap parameter map + * @return convert parameters place holders + */ + public static String convertParameterPlaceholders(String parameterString, Map parameterMap) { + if (StringUtils.isEmpty(parameterString)) { + return parameterString; + } + Date cronTime; + if (parameterMap != null && !parameterMap.isEmpty()) { + // replace variable ${} form,refers to the replacement of system variables and custom variables + parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); + } + if (parameterMap != null && null != parameterMap.get(PARAMETER_DATETIME)) { + //Get current time, schedule execute time + String cronTimeStr = parameterMap.get(PARAMETER_DATETIME); + cronTime = DateUtils.parse(cronTimeStr, PARAMETER_FORMAT_TIME); + } else { + cronTime = new Date(); + } + // replace time $[...] form, eg. $[yyyyMMdd] + if (cronTime != null) { + return dateTemplateParse(parameterString, cronTime); + } + return parameterString; + } + + /** + * new + * convert parameters place holders + * + * @param parameterString parameter + * @param parameterMap parameter map + * @return convert parameters place holders + */ + public static String convertParameterPlaceholders2(String parameterString, Map parameterMap) { + if (StringUtils.isEmpty(parameterString)) { + return parameterString; + } + //Get current time, schedule execute time + String cronTimeStr = parameterMap.get(PARAMETER_SHECDULE_TIME); + Date cronTime = null; + + if (StringUtils.isNotEmpty(cronTimeStr)) { + cronTime = DateUtils.parse(cronTimeStr, PARAMETER_FORMAT_TIME); + + } else { + cronTime = new Date(); + } + + // replace variable ${} form,refers to the replacement of system variables and custom variables + if (!parameterMap.isEmpty()) { + parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); + } + + // replace time $[...] form, eg. $[yyyyMMdd] + if (cronTime != null) { + return dateTemplateParse(parameterString, cronTime); + } + return parameterString; + } + + /** + * set in parameter + * + * @param index index + * @param stmt preparedstatement + * @param dataType data type + * @param value value + * @throws Exception errors + */ + public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value) throws Exception { + if (dataType.equals(DataType.VARCHAR)) { + stmt.setString(index, value); + } else if (dataType.equals(DataType.INTEGER)) { + stmt.setInt(index, Integer.parseInt(value)); + } else if (dataType.equals(DataType.LONG)) { + stmt.setLong(index, Long.parseLong(value)); + } else if (dataType.equals(DataType.FLOAT)) { + stmt.setFloat(index, Float.parseFloat(value)); + } else if (dataType.equals(DataType.DOUBLE)) { + stmt.setDouble(index, Double.parseDouble(value)); + } else if (dataType.equals(DataType.DATE)) { + stmt.setDate(index, java.sql.Date.valueOf(value)); + } else if (dataType.equals(DataType.TIME)) { + stmt.setString(index, value); + } else if (dataType.equals(DataType.TIMESTAMP)) { + stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value)); + } else if (dataType.equals(DataType.BOOLEAN)) { + stmt.setBoolean(index, Boolean.parseBoolean(value)); + } + } + + /** + * curing user define parameters + * + * @param globalParamMap global param map + * @param globalParamList global param list + * @param commandType command type + * @param scheduleTime schedule time + * @return curing user define parameters + */ + public static String curingGlobalParams(Map globalParamMap, List globalParamList, + CommandType commandType, Date scheduleTime) { + + if (globalParamList == null || globalParamList.isEmpty()) { + return null; + } + + Map globalMap = new HashMap<>(); + if (globalParamMap != null) { + globalMap.putAll(globalParamMap); + } + Map allParamMap = new HashMap<>(); + //If it is a complement, a complement time needs to be passed in, according to the task type + Map timeParams = BusinessTimeUtils + .getBusinessTime(commandType, scheduleTime); + + if (timeParams != null) { + allParamMap.putAll(timeParams); + } + + allParamMap.putAll(globalMap); + + Set> entries = allParamMap.entrySet(); + + Map resolveMap = new HashMap<>(); + for (Map.Entry entry : entries) { + String val = entry.getValue(); + if (val.startsWith("$")) { + String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap); + resolveMap.put(entry.getKey(), str); + } + } + globalMap.putAll(resolveMap); + + for (Property property : globalParamList) { + String val = globalMap.get(property.getProp()); + if (val != null) { + property.setValue(val); + } + } + return JSONUtils.toJsonString(globalParamList); + } + + /** + * $[yyyyMMdd] replace schedule time + */ + public static String replaceScheduleTime(String text, Date scheduleTime) { + Map paramsMap = new HashMap<>(); + //if getScheduleTime null ,is current date + if (null == scheduleTime) { + scheduleTime = new Date(); + } + + String dateTime = DateUtils.format(scheduleTime, PARAMETER_FORMAT_TIME); + Property p = new Property(); + p.setValue(dateTime); + p.setProp(PARAMETER_SHECDULE_TIME); + paramsMap.put(PARAMETER_SHECDULE_TIME, p); + text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap)); + + return text; + } + + /** + * format convert + * + * @param paramsMap params map + * @return Map of converted + * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert + */ + public static Map convert(Map paramsMap) { + Map map = new HashMap<>(); + Iterator> iter = paramsMap.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry en = iter.next(); + map.put(en.getKey(), en.getValue().getValue()); + } + return map; + } + + private static String dateTemplateParse(String templateStr, Date date) { + if (templateStr == null) { + return null; + } + Pattern pattern = Pattern.compile(DATE_PARSE_PATTERN); + + StringBuffer newValue = new StringBuffer(templateStr.length()); + + Matcher matcher = pattern.matcher(templateStr); + + while (matcher.find()) { + String key = matcher.group(1); + if (Pattern.matches(DATE_START_PATTERN, key)) { + continue; + } + String value = TimePlaceholderUtils.getPlaceHolderTime(key, date); + assert value != null; + matcher.appendReplacement(newValue, value); + } + + matcher.appendTail(newValue); + + return newValue.toString(); + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/PlaceholderUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/PlaceholderUtils.java new file mode 100644 index 0000000000..90ee18311a --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/PlaceholderUtils.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.paramparser; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * placeholder utils + */ +public class PlaceholderUtils { + + private static final Logger logger = LoggerFactory.getLogger(PlaceholderUtils.class); + + /** + * Prefix of the position to be replaced + */ + public static final String PLACEHOLDER_PREFIX = "${"; + + /** + * The suffix of the position to be replaced + */ + + public static final String PLACEHOLDER_SUFFIX = "}"; + + /** + * Replaces all placeholders of format {@code ${name}} with the value returned + * from the supplied {@link PropertyPlaceholderHelper.PlaceholderResolver}. + * + * @param value the value containing the placeholders to be replaced + * @param paramsMap placeholder data dictionary + * @param ignoreUnresolvablePlaceholders ignoreUnresolvablePlaceholders + * @return the supplied value with placeholders replaced inline + */ + public static String replacePlaceholders(String value, + Map paramsMap, + boolean ignoreUnresolvablePlaceholders) { + //replacement tool， parameter key will be replaced by value,if can't match , will throw an exception + PropertyPlaceholderHelper strictHelper = getPropertyPlaceholderHelper(false); + + //Non-strict replacement tool implementation, when the position to be replaced does not get the corresponding value, the current position is ignored, and the next position is replaced. + PropertyPlaceholderHelper nonStrictHelper = getPropertyPlaceholderHelper(true); + + PropertyPlaceholderHelper helper = (ignoreUnresolvablePlaceholders ? nonStrictHelper : strictHelper); + + //the PlaceholderResolver to use for replacement + return helper.replacePlaceholders(value, new PropertyPlaceholderResolver(value, paramsMap)); + } + + /** + * Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix. + * @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should + * be ignored ({@code true}) or cause an exception ({@code false}) + * @return PropertyPlaceholderHelper + */ + public static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { + + return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders); + } + + /** + * Placeholder replacement resolver + */ + private static class PropertyPlaceholderResolver implements PropertyPlaceholderHelper.PlaceholderResolver { + + private final String value; + + private final Map paramsMap; + + public PropertyPlaceholderResolver(String value, Map paramsMap) { + this.value = value; + this.paramsMap = paramsMap; + } + + @Override + public String resolvePlaceholder(String placeholderName) { + try { + return paramsMap.get(placeholderName); + } catch (Exception ex) { + logger.error("resolve placeholder '{}' in [ {} ]", placeholderName, value, ex); + return null; + } + } + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/PropertyPlaceholderHelper.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/PropertyPlaceholderHelper.java new file mode 100644 index 0000000000..7c571d6c63 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/PropertyPlaceholderHelper.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.paramparser; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for working with Strings that have placeholder values in them. A placeholder takes the form + * {@code ${name}}. Using {@code PropertyPlaceholderHelper} these placeholders can be substituted for + * user-supplied values.

Values for substitution can be supplied using a {@link Properties} instance or + * using a {@link PlaceholderResolver}. + * + * @author Juergen Hoeller + * @author Rob Harrop + * @since 3.0 + */ +public class PropertyPlaceholderHelper { + + private static final Logger logger = LoggerFactory.getLogger(PropertyPlaceholderHelper.class); + + private static final Map wellKnownSimplePrefixes = new HashMap(4); + + static { + wellKnownSimplePrefixes.put("}", "{"); + wellKnownSimplePrefixes.put("]", "["); + wellKnownSimplePrefixes.put(")", "("); + } + + + private final String placeholderPrefix; + + private final String placeholderSuffix; + + private final String simplePrefix; + + private final String valueSeparator; + + private final boolean ignoreUnresolvablePlaceholders; + + /** + * Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix. + * Unresolvable placeholders are ignored. + * @param placeholderPrefix the prefix that denotes the start of a placeholder + * @param placeholderSuffix the suffix that denotes the end of a placeholder + */ + public PropertyPlaceholderHelper(String placeholderPrefix, String placeholderSuffix) { + this(placeholderPrefix, placeholderSuffix, null, true); + } + + /** + * Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix. + * @param placeholderPrefix the prefix that denotes the start of a placeholder + * @param placeholderSuffix the suffix that denotes the end of a placeholder + * @param valueSeparator the separating character between the placeholder variable + * and the associated default value, if any + * @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should + * be ignored ({@code true}) or cause an exception ({@code false}) + */ + public PropertyPlaceholderHelper(String placeholderPrefix, String placeholderSuffix, + String valueSeparator, boolean ignoreUnresolvablePlaceholders) { + + notNull(placeholderPrefix, "'placeholderPrefix' must not be null"); + notNull(placeholderSuffix, "'placeholderSuffix' must not be null"); + this.placeholderPrefix = placeholderPrefix; + this.placeholderSuffix = placeholderSuffix; + String simplePrefixForSuffix = wellKnownSimplePrefixes.get(this.placeholderSuffix); + if (simplePrefixForSuffix != null && this.placeholderPrefix.endsWith(simplePrefixForSuffix)) { + this.simplePrefix = simplePrefixForSuffix; + } + else { + this.simplePrefix = this.placeholderPrefix; + } + this.valueSeparator = valueSeparator; + this.ignoreUnresolvablePlaceholders = ignoreUnresolvablePlaceholders; + } + + /** + * Replaces all placeholders of format {@code ${name}} with the corresponding + * property from the supplied {@link Properties}. + * @param value the value containing the placeholders to be replaced + * @param properties the {@code Properties} to use for replacement + * @return the supplied value with placeholders replaced inline + */ + public String replacePlaceholders(String value, final Properties properties) { + notNull(properties, "'properties' must not be null"); + return replacePlaceholders(value, new PlaceholderResolver() { + @Override + public String resolvePlaceholder(String placeholderName) { + return properties.getProperty(placeholderName); + } + }); + } + + /** + * Replaces all placeholders of format {@code ${name}} with the value returned + * from the supplied {@link PlaceholderResolver}. + * @param value the value containing the placeholders to be replaced + * @param placeholderResolver the {@code PlaceholderResolver} to use for replacement + * @return the supplied value with placeholders replaced inline + */ + public String replacePlaceholders(String value, PlaceholderResolver placeholderResolver) { + notNull(value, "'value' must not be null"); + return parseStringValue(value, placeholderResolver, new HashSet()); + } + + protected String parseStringValue( + String value, PlaceholderResolver placeholderResolver, Set visitedPlaceholders) { + + StringBuilder result = new StringBuilder(value); + + int startIndex = value.indexOf(this.placeholderPrefix); + while (startIndex != -1) { + int endIndex = findPlaceholderEndIndex(result, startIndex); + if (endIndex != -1) { + String placeholder = result.substring(startIndex + this.placeholderPrefix.length(), endIndex); + String originalPlaceholder = placeholder; + if (!visitedPlaceholders.add(originalPlaceholder)) { + throw new IllegalArgumentException( + "Circular placeholder reference '" + originalPlaceholder + "' in property definitions"); + } + // Recursive invocation, parsing placeholders contained in the placeholder key. + placeholder = parseStringValue(placeholder, placeholderResolver, visitedPlaceholders); + // Now obtain the value for the fully resolved key... + String propVal = placeholderResolver.resolvePlaceholder(placeholder); + if (propVal == null && this.valueSeparator != null) { + int separatorIndex = placeholder.indexOf(this.valueSeparator); + if (separatorIndex != -1) { + String actualPlaceholder = placeholder.substring(0, separatorIndex); + String defaultValue = placeholder.substring(separatorIndex + this.valueSeparator.length()); + propVal = placeholderResolver.resolvePlaceholder(actualPlaceholder); + if (propVal == null) { + propVal = defaultValue; + } + } + } + if (propVal != null) { + // Recursive invocation, parsing placeholders contained in the + // previously resolved placeholder value. + propVal = parseStringValue(propVal, placeholderResolver, visitedPlaceholders); + result.replace(startIndex, endIndex + this.placeholderSuffix.length(), propVal); + if (logger.isTraceEnabled()) { + logger.trace("Resolved placeholder '" + placeholder + "'"); + } + startIndex = result.indexOf(this.placeholderPrefix, startIndex + propVal.length()); + } + else if (this.ignoreUnresolvablePlaceholders) { + // Proceed with unprocessed value. + startIndex = result.indexOf(this.placeholderPrefix, endIndex + this.placeholderSuffix.length()); + } + else { + throw new IllegalArgumentException("Could not resolve placeholder '" + + placeholder + "'" + " in value \"" + value + "\""); + } + visitedPlaceholders.remove(originalPlaceholder); + } + else { + startIndex = -1; + } + } + + return result.toString(); + } + + private int findPlaceholderEndIndex(CharSequence buf, int startIndex) { + int index = startIndex + this.placeholderPrefix.length(); + int withinNestedPlaceholder = 0; + while (index < buf.length()) { + if (substringMatch(buf, index, this.placeholderSuffix)) { + if (withinNestedPlaceholder > 0) { + withinNestedPlaceholder--; + index = index + this.placeholderSuffix.length(); + } + else { + return index; + } + } + else if (substringMatch(buf, index, this.simplePrefix)) { + withinNestedPlaceholder++; + index = index + this.simplePrefix.length(); + } + else { + index++; + } + } + return -1; + } + + /** + * Strategy interface used to resolve replacement values for placeholders contained in Strings. + */ + public interface PlaceholderResolver { + + /** + * Resolve the supplied placeholder name to the replacement value. + * @param placeholderName the name of the placeholder to resolve + * @return the replacement value, or {@code null} if no replacement is to be made + */ + String resolvePlaceholder(String placeholderName); + } + + /** + * Test whether the given string matches the given substring + * at the given index. + * @param str the original string (or StringBuilder) + * @param index the index in the original string to start matching against + * @param substring the substring to match at the given index + * @return whether the given string matches the given substring + */ + public static boolean substringMatch(CharSequence str, int index, CharSequence substring) { + for (int j = 0; j < substring.length(); j++) { + int i = index + j; + if (i >= str.length() || str.charAt(i) != substring.charAt(j)) { + return false; + } + } + return true; + } + + /** + * Assert that an object is not {@code null}. + * 

Assert.notNull(clazz, "The class must not be null");
+ * @param object the object to check + * @param message the exception message to use if the assertion fails + * @throws IllegalArgumentException if the object is {@code null} + */ + public static void notNull(Object object, String message) { + if (object == null) { + throw new IllegalArgumentException(message); + } + } + +} + diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/TimePlaceholderUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/TimePlaceholderUtils.java new file mode 100644 index 0000000000..da72089f92 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/paramparser/TimePlaceholderUtils.java @@ -0,0 +1,570 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.paramparser; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.ADD_CHAR; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.ADD_MONTHS; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.ADD_STRING; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DIVISION_CHAR; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DIVISION_STRING; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.LEFT_BRACE_CHAR; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.LEFT_BRACE_STRING; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.MONTH_BEGIN; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.MONTH_END; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.MULTIPLY_CHAR; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.MULTIPLY_STRING; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.N; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.P; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.PARAMETER_FORMAT_TIME; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.RIGHT_BRACE_CHAR; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SUBTRACT_CHAR; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SUBTRACT_STRING; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.TIMESTAMP; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.WEEK_BEGIN; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.WEEK_END; +import static org.apache.dolphinscheduler.spi.utils.DateUtils.addDays; +import static org.apache.dolphinscheduler.spi.utils.DateUtils.addMinutes; +import static org.apache.dolphinscheduler.spi.utils.DateUtils.addMonths; + +import org.apache.dolphinscheduler.spi.utils.DateUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * time place holder utils + */ +public class TimePlaceholderUtils { + private static final Logger logger = LoggerFactory.getLogger(TimePlaceholderUtils.class); + + /** + * Prefix of the position to be replaced + */ + public static final String PLACEHOLDER_PREFIX = "$["; + + /** + * The suffix of the position to be replaced + */ + public static final String PLACEHOLDER_SUFFIX = "]"; + + /** + * Replaces all placeholders of format {@code ${name}} with the value returned + * from the supplied {@link PropertyPlaceholderHelper.PlaceholderResolver}. + * + * @param value the value containing the placeholders to be replaced + * @param date custom date + * @param ignoreUnresolvablePlaceholders ignore unresolvable placeholders + * @return the supplied value with placeholders replaced inline + */ + public static String replacePlaceholders(String value, Date date, boolean ignoreUnresolvablePlaceholders) { + PropertyPlaceholderHelper strictHelper = getPropertyPlaceholderHelper(false); + PropertyPlaceholderHelper nonStrictHelper = getPropertyPlaceholderHelper(true); + + PropertyPlaceholderHelper helper = (ignoreUnresolvablePlaceholders ? nonStrictHelper : strictHelper); + return helper.replacePlaceholders(value, new TimePlaceholderResolver(value, date)); + } + + /** + * Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix. + * + * @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should + * be ignored ({@code true}) or cause an exception ({@code false}) + */ + private static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { + return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders); + } + + /** + * calculate expression's value + * + * @param expression expression + * @return expression's value + */ + public static Integer calculate(String expression) { + expression = StringUtils.trim(expression); + expression = convert(expression); + + List result = string2List(expression); + result = convert2SuffixList(result); + + return calculate(result); + } + + /** + * Change the sign in the expression to P (positive) N (negative) + * + * @param expression + * @return eg. "-3+-6*(+8)-(-5) -> S3+S6*(P8)-(S5)" + */ + private static String convert(String expression) { + char[] arr = expression.toCharArray(); + + for (int i = 0; i < arr.length; i++) { + if (arr[i] == SUBTRACT_CHAR) { + if (i == 0) { + arr[i] = N; + } else { + char c = arr[i - 1]; + if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR || c == LEFT_BRACE_CHAR) { + arr[i] = N; + } + } + } else if (arr[i] == ADD_CHAR) { + if (i == 0) { + arr[i] = P; + } else { + char c = arr[i - 1]; + if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR || c == LEFT_BRACE_CHAR) { + arr[i] = P; + } + } + } + } + + return new String(arr); + } + + /** + * to suffix expression + * + * @param srcList + * @return + */ + private static List convert2SuffixList(List srcList) { + List result = new ArrayList<>(); + Stack stack = new Stack<>(); + + for (int i = 0; i < srcList.size(); i++) { + if (Character.isDigit(srcList.get(i).charAt(0))) { + result.add(srcList.get(i)); + } else { + switch (srcList.get(i).charAt(0)) { + case LEFT_BRACE_CHAR: + stack.push(srcList.get(i)); + break; + case RIGHT_BRACE_CHAR: + while (!LEFT_BRACE_STRING.equals(stack.peek())) { + result.add(stack.pop()); + } + stack.pop(); + break; + default: + while (!stack.isEmpty() && compare(stack.peek(), srcList.get(i))) { + result.add(stack.pop()); + } + stack.push(srcList.get(i)); + break; + } + } + } + + while (!stack.isEmpty()) { + result.add(stack.pop()); + } + + return result; + } + + /** + * Calculate the suffix expression + * + * @param result + * @return + */ + private static Integer calculate(List result) { + Stack stack = new Stack<>(); + for (int i = 0; i < result.size(); i++) { + if (Character.isDigit(result.get(i).charAt(0))) { + stack.push(Integer.parseInt(result.get(i))); + } else { + Integer backInt = stack.pop(); + Integer frontInt = 0; + char op = result.get(i).charAt(0); + + if (!(op == P || op == N)) { + frontInt = stack.pop(); + } + + Integer res = 0; + switch (result.get(i).charAt(0)) { + case P: + res = frontInt + backInt; + break; + case N: + res = frontInt - backInt; + break; + case ADD_CHAR: + res = frontInt + backInt; + break; + case SUBTRACT_CHAR: + res = frontInt - backInt; + break; + case MULTIPLY_CHAR: + res = frontInt * backInt; + break; + case DIVISION_CHAR: + res = frontInt / backInt; + break; + default: + break; + } + stack.push(res); + } + } + + return stack.pop(); + } + + /** + * string to list + * + * @param expression + * @return list + */ + private static List string2List(String expression) { + List result = new ArrayList<>(); + String num = ""; + for (int i = 0; i < expression.length(); i++) { + if (Character.isDigit(expression.charAt(i))) { + num = num + expression.charAt(i); + } else { + if (!num.isEmpty()) { + result.add(num); + } + result.add(expression.charAt(i) + ""); + num = ""; + } + } + + if (!num.isEmpty()) { + result.add(num); + } + + return result; + } + + /** + * compare loginUser level + * + * @param peek + * @param cur + * @return true or false + */ + private static boolean compare(String peek, String cur) { + if (MULTIPLY_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { + return true; + } else if (DIVISION_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { + return true; + } else if (ADD_STRING.equals(peek) && (ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { + return true; + } else { + return SUBTRACT_STRING.equals(peek) && (ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur)); + } + + } + + /** + * Placeholder replacement resolver + */ + private static class TimePlaceholderResolver implements + PropertyPlaceholderHelper.PlaceholderResolver { + + private final String value; + + private final Date date; + + public TimePlaceholderResolver(String value, Date date) { + this.value = value; + this.date = date; + } + + @Override + public String resolvePlaceholder(String placeholderName) { + try { + return calculateTime(placeholderName, date); + } catch (Exception ex) { + logger.error("resolve placeholder '{}' in [ {} ]", placeholderName, value, ex); + return null; + } + } + } + + /** + * return the formatted date according to the corresponding date format + * + * @param expression date expression + * @param date date + * @return reformat date + */ + public static String getPlaceHolderTime(String expression, Date date) { + if (StringUtils.isBlank(expression)) { + return null; + } + if (null == date) { + return null; + } + return calculateTime(expression, date); + } + + /** + * calculate time + * + * @param date date + * @return calculate time + */ + private static String calculateTime(String expression, Date date) { + // After N years: $[add_months(yyyyMMdd,12*N)], the first N months: $[add_months(yyyyMMdd,-N)], etc + String value; + + try { + if (expression.startsWith(TIMESTAMP)) { + String timeExpression = expression.substring(TIMESTAMP.length() + 1, expression.length() - 1); + + Map.Entry entry = calcTimeExpression(timeExpression, date); + + String dateStr = DateUtils.format(entry.getKey(), entry.getValue()); + + Date timestamp = DateUtils.parse(dateStr, PARAMETER_FORMAT_TIME); + + value = String.valueOf(timestamp.getTime() / 1000); + } else { + Map.Entry entry = calcTimeExpression(expression, date); + value = DateUtils.format(entry.getKey(), entry.getValue()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw e; + } + + return value; + } + + /** + * calculate time expresstion + * + * @param expression expresstion + * @param date date + * @return map with date, date format + */ + public static Map.Entry calcTimeExpression(String expression, Date date) { + Map.Entry resultEntry; + + if (expression.startsWith(ADD_MONTHS)) { + resultEntry = calcMonths(expression, date); + } else if (expression.startsWith(MONTH_BEGIN)) { + resultEntry = calcMonthBegin(expression, date); + } else if (expression.startsWith(MONTH_END)) { + resultEntry = calcMonthEnd(expression, date); + } else if (expression.startsWith(WEEK_BEGIN)) { + resultEntry = calcWeekStart(expression, date); + } else if (expression.startsWith(WEEK_END)) { + resultEntry = calcWeekEnd(expression, date); + } else { + resultEntry = calcMinutes(expression, date); + } + + return resultEntry; + } + + /** + * get first day of month + * + * @param expression expresstion + * @param date date + * @return first day of month + */ + public static Map.Entry calcMonthBegin(String expression, Date date) { + String addMonthExpr = expression.substring(MONTH_BEGIN.length() + 1, expression.length() - 1); + String[] params = addMonthExpr.split(COMMA); + + if (params.length == 2) { + String dateFormat = params[0]; + String dayExpr = params[1]; + Integer day = calculate(dayExpr); + Date targetDate = DateUtils.getFirstDayOfMonth(date); + targetDate = addDays(targetDate, day); + + return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); + } + + throw new RuntimeException("expression not valid"); + } + + /** + * get last day of month + * + * @param expression expresstion + * @param date date + * @return last day of month + */ + public static Map.Entry calcMonthEnd(String expression, Date date) { + String addMonthExpr = expression.substring(MONTH_END.length() + 1, expression.length() - 1); + String[] params = addMonthExpr.split(COMMA); + + if (params.length == 2) { + String dateFormat = params[0]; + String dayExpr = params[1]; + Integer day = calculate(dayExpr); + Date targetDate = DateUtils.getLastDayOfMonth(date); + targetDate = addDays(targetDate, day); + + return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); + } + + throw new RuntimeException("expression not valid"); + } + + /** + * get first day of week + * + * @param expression expresstion + * @param date date + * @return monday + */ + public static Map.Entry calcWeekStart(String expression, Date date) { + String addMonthExpr = expression.substring(WEEK_BEGIN.length() + 1, expression.length() - 1); + String[] params = addMonthExpr.split(COMMA); + + if (params.length == 2) { + String dateFormat = params[0]; + String dayExpr = params[1]; + Integer day = calculate(dayExpr); + Date targetDate = DateUtils.getMonday(date); + targetDate = addDays(targetDate, day); + return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); + } + + throw new RuntimeException("expression not valid"); + } + + /** + * get last day of week + * + * @param expression expresstion + * @param date date + * @return last day of week + */ + public static Map.Entry calcWeekEnd(String expression, Date date) { + String addMonthExpr = expression.substring(WEEK_END.length() + 1, expression.length() - 1); + String[] params = addMonthExpr.split(COMMA); + + if (params.length == 2) { + String dateFormat = params[0]; + String dayExpr = params[1]; + Integer day = calculate(dayExpr); + Date targetDate = DateUtils.getSunday(date); + targetDate = addDays(targetDate, day); + + return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); + } + + throw new RuntimeException("Expression not valid"); + } + + /** + * calc months expression + * + * @param expression expresstion + * @param date date + * @return calc months + */ + public static Map.Entry calcMonths(String expression, Date date) { + String addMonthExpr = expression.substring(ADD_MONTHS.length() + 1, expression.length() - 1); + String[] params = addMonthExpr.split(COMMA); + + if (params.length == 2) { + String dateFormat = params[0]; + String monthExpr = params[1]; + Integer addMonth = calculate(monthExpr); + Date targetDate = addMonths(date, addMonth); + + return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); + } + + throw new RuntimeException("expression not valid"); + } + + /** + * calculate time expression + * + * @param expression expresstion + * @param date date + * @return calculate time expression with date,format + */ + public static Map.Entry calcMinutes(String expression, Date date) { + if (expression.contains("+")) { + int index = expression.lastIndexOf('+'); + + if (Character.isDigit(expression.charAt(index + 1))) { + String addMinuteExpr = expression.substring(index + 1); + Date targetDate = addMinutes(date, calcMinutes(addMinuteExpr)); + String dateFormat = expression.substring(0, index); + + return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); + } + } else if (expression.contains("-")) { + int index = expression.lastIndexOf('-'); + + if (Character.isDigit(expression.charAt(index + 1))) { + String addMinuteExpr = expression.substring(index + 1); + Date targetDate = addMinutes(date, 0 - calcMinutes(addMinuteExpr)); + String dateFormat = expression.substring(0, index); + + return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); + } + + // yyyy-MM-dd/HH:mm:ss + return new AbstractMap.SimpleImmutableEntry<>(date, expression); + } + + // $[HHmmss] + return new AbstractMap.SimpleImmutableEntry<>(date, expression); + } + + /** + * calculate need minutes + * + * @param minuteExpression minute expression + * @return calculate need minutes + */ + public static Integer calcMinutes(String minuteExpression) { + int index = minuteExpression.indexOf('/'); + + String calcExpression; + + if (index == -1) { + calcExpression = String.format("60*24*(%s)", minuteExpression); + } else { + + calcExpression = String.format("60*24*(%s)%s", minuteExpression.substring(0, index), + minuteExpression.substring(index)); + } + + return calculate(calcExpression); + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/DataxTaskRequest.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/DataxTaskRequest.java new file mode 100644 index 0000000000..8afc397398 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/DataxTaskRequest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.request; + +/** + * DataX Task ExecutionContext + * to master/worker task transport + */ +public class DataxTaskRequest extends TaskRequest { + + /** + * dataSourceId + */ + private int dataSourceId; + + /** + * sourcetype + */ + private int sourcetype; + + /** + * sourceConnectionParams + */ + private String sourceConnectionParams; + + /** + * dataTargetId + */ + private int dataTargetId; + + /** + * targetType + */ + private int targetType; + + /** + * targetConnectionParams + */ + private String targetConnectionParams; + + public int getDataSourceId() { + return dataSourceId; + } + + public void setDataSourceId(int dataSourceId) { + this.dataSourceId = dataSourceId; + } + + public int getSourcetype() { + return sourcetype; + } + + public void setSourcetype(int sourcetype) { + this.sourcetype = sourcetype; + } + + public String getSourceConnectionParams() { + return sourceConnectionParams; + } + + public void setSourceConnectionParams(String sourceConnectionParams) { + this.sourceConnectionParams = sourceConnectionParams; + } + + public int getDataTargetId() { + return dataTargetId; + } + + public void setDataTargetId(int dataTargetId) { + this.dataTargetId = dataTargetId; + } + + public int getTargetType() { + return targetType; + } + + public void setTargetType(int targetType) { + this.targetType = targetType; + } + + public String getTargetConnectionParams() { + return targetConnectionParams; + } + + public void setTargetConnectionParams(String targetConnectionParams) { + this.targetConnectionParams = targetConnectionParams; + } + + @Override + public String toString() { + return "DataxTaskExecutionContext{" + + "dataSourceId=" + dataSourceId + + ", sourcetype=" + sourcetype + + ", sourceConnectionParams='" + sourceConnectionParams + '\'' + + ", dataTargetId=" + dataTargetId + + ", targetType=" + targetType + + ", targetConnectionParams='" + targetConnectionParams + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/ProcedureTaskRequest.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/ProcedureTaskRequest.java new file mode 100644 index 0000000000..b043d7f1d4 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/ProcedureTaskRequest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.request; + +/** + * Procedure Task ExecutionContext + * to master/worker task transport + */ +public class ProcedureTaskRequest extends TaskRequest { + + /** + * connectionParams + */ + private String connectionParams; + + public String getConnectionParams() { + return connectionParams; + } + + public void setConnectionParams(String connectionParams) { + this.connectionParams = connectionParams; + } + + @Override + public String toString() { + return "ProcedureTaskExecutionContext{" + + "connectionParams='" + connectionParams + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/SQLTaskRequest.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/SQLTaskRequest.java new file mode 100644 index 0000000000..0608af2fee --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/SQLTaskRequest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.request; + +import org.apache.dolphinscheduler.spi.task.UdfFuncBean; +import org.apache.dolphinscheduler.spi.task.UdfFuncBean.UdfFuncDeserializer; + +import java.util.Map; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +/** + * SQL Task ExecutionContext + * to master/worker task transport + */ +public class SQLTaskRequest extends TaskRequest { + + /** + * warningGroupId + */ + private int warningGroupId; + + /** + * connectionParams + */ + private String connectionParams; + /** + * udf function tenant code map + */ + @JsonDeserialize(keyUsing = UdfFuncDeserializer.class) + private Map udfFuncTenantCodeMap; + + public int getWarningGroupId() { + return warningGroupId; + } + + public void setWarningGroupId(int warningGroupId) { + this.warningGroupId = warningGroupId; + } + + public Map getUdfFuncTenantCodeMap() { + return udfFuncTenantCodeMap; + } + + public void setUdfFuncTenantCodeMap(Map udfFuncTenantCodeMap) { + this.udfFuncTenantCodeMap = udfFuncTenantCodeMap; + } + + public String getConnectionParams() { + return connectionParams; + } + + public void setConnectionParams(String connectionParams) { + this.connectionParams = connectionParams; + } + + @Override + public String toString() { + return "SQLTaskExecutionContext{" + + "warningGroupId=" + warningGroupId + + ", connectionParams='" + connectionParams + '\'' + + ", udfFuncTenantCodeMap=" + udfFuncTenantCodeMap + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/SqoopTaskRequest.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/SqoopTaskRequest.java new file mode 100644 index 0000000000..69f8cf8d66 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/SqoopTaskRequest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.task.request; + +/** + * Sqoop Task ExecutionContext + * to master/worker task transport + */ +public class SqoopTaskRequest extends TaskRequest { + + /** + * dataSourceId + */ + private int dataSourceId; + + /** + * sourcetype + */ + private int sourcetype; + + /** + * sourceConnectionParams + */ + private String sourceConnectionParams; + + /** + * dataTargetId + */ + private int dataTargetId; + + /** + * targetType + */ + private int targetType; + + /** + * targetConnectionParams + */ + private String targetConnectionParams; + + public int getDataSourceId() { + return dataSourceId; + } + + public void setDataSourceId(int dataSourceId) { + this.dataSourceId = dataSourceId; + } + + public int getSourcetype() { + return sourcetype; + } + + public void setSourcetype(int sourcetype) { + this.sourcetype = sourcetype; + } + + public String getSourceConnectionParams() { + return sourceConnectionParams; + } + + public void setSourceConnectionParams(String sourceConnectionParams) { + this.sourceConnectionParams = sourceConnectionParams; + } + + public int getDataTargetId() { + return dataTargetId; + } + + public void setDataTargetId(int dataTargetId) { + this.dataTargetId = dataTargetId; + } + + public int getTargetType() { + return targetType; + } + + public void setTargetType(int targetType) { + this.targetType = targetType; + } + + public String getTargetConnectionParams() { + return targetConnectionParams; + } + + public void setTargetConnectionParams(String targetConnectionParams) { + this.targetConnectionParams = targetConnectionParams; + } + + @Override + public String toString() { + return "SqoopTaskExecutionContext{" + + "dataSourceId=" + dataSourceId + + ", sourcetype=" + sourcetype + + ", sourceConnectionParams='" + sourceConnectionParams + '\'' + + ", dataTargetId=" + dataTargetId + + ", targetType=" + targetType + + ", targetConnectionParams='" + targetConnectionParams + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskRequest.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/TaskRequest.java similarity index 95% rename from dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskRequest.java rename to dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/TaskRequest.java index 1394c27625..0de96e1734 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskRequest.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/request/TaskRequest.java @@ -15,17 +15,21 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.spi.task; +package org.apache.dolphinscheduler.spi.task.request; + +import org.apache.dolphinscheduler.spi.enums.TaskTimeoutStrategy; +import org.apache.dolphinscheduler.spi.task.Property; import java.util.Date; import java.util.Map; import com.fasterxml.jackson.annotation.JsonFormat; +/** + * to master/worker task transport + */ public class TaskRequest { - - /** * task id */ @@ -157,7 +161,7 @@ public class TaskRequest { /** * task timeout strategy */ - private int taskTimeoutStrategy; + private TaskTimeoutStrategy taskTimeoutStrategy; /** * task timeout @@ -392,11 +396,11 @@ public class TaskRequest { this.taskAppId = taskAppId; } - public int getTaskTimeoutStrategy() { + public TaskTimeoutStrategy getTaskTimeoutStrategy() { return taskTimeoutStrategy; } - public void setTaskTimeoutStrategy(int taskTimeoutStrategy) { + public void setTaskTimeoutStrategy(TaskTimeoutStrategy taskTimeoutStrategy) { this.taskTimeoutStrategy = taskTimeoutStrategy; } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/CommonUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/CommonUtils.java new file mode 100644 index 0000000000..9017655d1c --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/CommonUtils.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.utils; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JAVA_SECURITY_KRB5_CONF; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JAVA_SECURITY_KRB5_CONF_PATH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.KERBEROS; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.LOGIN_USER_KEY_TAB_PATH; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.LOGIN_USER_KEY_TAB_USERNAME; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.RESOURCE_STORAGE_TYPE; + +import org.apache.dolphinscheduler.spi.enums.ResUploadType; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.IOException; + +/** + * common utils + */ +public class CommonUtils { + + private CommonUtils() { + throw new UnsupportedOperationException("Construct CommonUtils"); + } + + /** + * if upload resource is HDFS and kerberos startup is true , else false + * + * @return true if upload resource is HDFS and kerberos startup + */ + public static boolean getKerberosStartupState() { + String resUploadStartupType = PropertyUtils.getUpperCaseString(RESOURCE_STORAGE_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + Boolean kerberosStartupState = PropertyUtils.getBoolean(HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); + return resUploadType == ResUploadType.HDFS && kerberosStartupState; + } + + /** + * load kerberos configuration + * + * @param configuration + * @return load kerberos config return true + * @throws IOException errors + */ + public static boolean loadKerberosConf(Configuration configuration) throws IOException { + return loadKerberosConf(PropertyUtils.getString(JAVA_SECURITY_KRB5_CONF_PATH), + PropertyUtils.getString(LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(LOGIN_USER_KEY_TAB_PATH), configuration); + } + + /** + * load kerberos configuration + * + * @param javaSecurityKrb5Conf javaSecurityKrb5Conf + * @param loginUserKeytabUsername loginUserKeytabUsername + * @param loginUserKeytabPath loginUserKeytabPath + * @throws IOException errors + */ + public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws IOException { + loadKerberosConf(javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath, new Configuration()); + } + + /** + * load kerberos configuration + * + * @param javaSecurityKrb5Conf javaSecurityKrb5Conf + * @param loginUserKeytabUsername loginUserKeytabUsername + * @param loginUserKeytabPath loginUserKeytabPath + * @param configuration configuration + * @return load kerberos config return true + * @throws IOException errors + */ + public static boolean loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath, Configuration configuration) throws IOException { + if (CommonUtils.getKerberosStartupState()) { + System.setProperty(JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(JAVA_SECURITY_KRB5_CONF_PATH))); + configuration.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(StringUtils.defaultIfBlank(loginUserKeytabUsername, PropertyUtils.getString(LOGIN_USER_KEY_TAB_USERNAME)), + StringUtils.defaultIfBlank(loginUserKeytabPath, PropertyUtils.getString(LOGIN_USER_KEY_TAB_PATH))); + return true; + } + return false; + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java index d653d4e435..a3a8b77564 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java @@ -52,4 +52,24 @@ public class Constants { /** string no */ public static final String STRING_NO = "NO"; + /** + * common properties path + */ + public static final String COMMON_PROPERTIES_PATH = "/common.properties"; + + /** + * date format of yyyy-MM-dd HH:mm:ss + */ + public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss"; + + /** + * date format of yyyyMMddHHmmss + */ + public static final String YYYYMMDDHHMMSS = "yyyyMMddHHmmss"; + + /** + * date format of yyyyMMddHHmmssSSS + */ + public static final String YYYYMMDDHHMMSSSSS = "yyyyMMddHHmmssSSS"; + } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java new file mode 100644 index 0000000000..9e63498645 --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java @@ -0,0 +1,622 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.utils; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Calendar; +import java.util.Date; +import java.util.Objects; +import java.util.TimeZone; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * date utils + */ +public class DateUtils { + + static final long C0 = 1L; + static final long C1 = C0 * 1000L; + static final long C2 = C1 * 1000L; + static final long C3 = C2 * 1000L; + static final long C4 = C3 * 60L; + static final long C5 = C4 * 60L; + static final long C6 = C5 * 24L; + + /** + * a default datetime formatter for the timestamp + */ + private static final DateTimeFormatter DEFAULT_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); + + private DateUtils() { + throw new UnsupportedOperationException("Construct DateUtils"); + } + + /** + * @param timeMillis timeMillis like System.currentTimeMillis() + * @return string formatted as yyyy-MM-dd HH:mm:ss + */ + public static String formatTimeStamp(long timeMillis) { + return formatTimeStamp(timeMillis, DEFAULT_DATETIME_FORMATTER); + } + + /** + * @param timeMillis timeMillis like System.currentTimeMillis() + * @param dateTimeFormatter expect formatter, like yyyy-MM-dd HH:mm:ss + * @return formatted string + */ + public static String formatTimeStamp(long timeMillis, DateTimeFormatter dateTimeFormatter) { + Objects.requireNonNull(dateTimeFormatter); + return dateTimeFormatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeMillis), + ZoneId.systemDefault())); + } + + /** + * date to local datetime + * + * @param date date + * @return local datetime + */ + private static LocalDateTime date2LocalDateTime(Date date) { + return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + } + + /** + * local datetime to date + * + * @param localDateTime local datetime + * @return date + */ + private static Date localDateTime2Date(LocalDateTime localDateTime) { + Instant instant = localDateTime.atZone(ZoneId.systemDefault()).toInstant(); + return Date.from(instant); + } + + /** + * get current date str + * + * @return date string + */ + public static String getCurrentTime() { + return getCurrentTime(Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * get the date string in the specified format of the current time + * + * @param format date format + * @return date string + */ + public static String getCurrentTime(String format) { + return LocalDateTime.now().format(DateTimeFormatter.ofPattern(format)); + } + + /** + * get the formatted date string + * + * @param date date + * @param format e.g. yyyy-MM-dd HH:mm:ss + * @return date string + */ + public static String format(Date date, String format) { + return format(date2LocalDateTime(date), format); + } + + /** + * get the formatted date string + * + * @param localDateTime local data time + * @param format yyyy-MM-dd HH:mm:ss + * @return date string + */ + public static String format(LocalDateTime localDateTime, String format) { + return localDateTime.format(DateTimeFormatter.ofPattern(format)); + } + + /** + * convert time to yyyy-MM-dd HH:mm:ss format + * + * @param date date + * @return date string + */ + public static String dateToString(Date date) { + return format(date, Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * convert string to date and time + * + * @param date date + * @param format format + * @return date + */ + public static Date parse(String date, String format) { + try { + LocalDateTime ldt = LocalDateTime.parse(date, DateTimeFormatter.ofPattern(format)); + return localDateTime2Date(ldt); + } catch (Exception e) { + logger.error("error while parse date:" + date, e); + } + return null; + } + + /** + * convert date str to yyyy-MM-dd HH:mm:ss format + * + * @param str date string + * @return yyyy-MM-dd HH:mm:ss format + */ + public static Date stringToDate(String str) { + return parse(str, Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * get seconds between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ seconds + */ + public static long differSec(Date d1, Date d2) { + if (d1 == null || d2 == null) { + return 0; + } + return (long) Math.ceil(differMs(d1, d2) / 1000.0); + } + + /** + * get ms between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ ms + */ + public static long differMs(Date d1, Date d2) { + return Math.abs(d1.getTime() - d2.getTime()); + } + + /** + * get hours between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ hours + */ + public static long diffHours(Date d1, Date d2) { + return (long) Math.ceil(diffMin(d1, d2) / 60.0); + } + + /** + * get minutes between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ minutes + */ + public static long diffMin(Date d1, Date d2) { + return (long) Math.ceil(differSec(d1, d2) / 60.0); + } + + /** + * get the date of the specified date in the days before and after + * + * @param date date + * @param day day + * @return the date of the specified date in the days before and after + */ + public static Date getSomeDay(Date date, int day) { + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + calendar.add(Calendar.DATE, day); + return calendar.getTime(); + } + + /** + * get the hour of day. + * + * @param date date + * @return hour of day + */ + public static int getHourIndex(Date date) { + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + return calendar.get(Calendar.HOUR_OF_DAY); + } + + /** + * compare two dates + * + * @param future future date + * @param old old date + * @return true if future time greater than old time + */ + public static boolean compare(Date future, Date old) { + return future.getTime() > old.getTime(); + } + + /** + * convert schedule string to date + * + * @param schedule schedule + * @return convert schedule string to date + */ + public static Date getScheduleDate(String schedule) { + return stringToDate(schedule); + } + + /** + * format time to readable + * + * @param ms ms + * @return format time + */ + public static String format2Readable(long ms) { + + long days = MILLISECONDS.toDays(ms); + long hours = MILLISECONDS.toDurationHours(ms); + long minutes = MILLISECONDS.toDurationMinutes(ms); + long seconds = MILLISECONDS.toDurationSeconds(ms); + + return String.format("%02d %02d:%02d:%02d", days, hours, minutes, seconds); + + } + + /** + * format time to duration + * + * @param d1 d1 + * @param d2 d2 + * @return format time + */ + public static String format2Duration(Date d1, Date d2) { + if (d1 == null || d2 == null) { + return null; + } + return format2Duration(differMs(d1, d2)); + } + + /** + * format time to duration + * + * @param ms ms + * @return format time + */ + public static String format2Duration(long ms) { + + long days = MILLISECONDS.toDays(ms); + long hours = MILLISECONDS.toDurationHours(ms); + long minutes = MILLISECONDS.toDurationMinutes(ms); + long seconds = MILLISECONDS.toDurationSeconds(ms); + + StringBuilder strBuilder = new StringBuilder(); + strBuilder = days > 0 ? strBuilder.append(days).append("d").append(" ") : strBuilder; + strBuilder = hours > 0 ? strBuilder.append(hours).append("h").append(" ") : strBuilder; + strBuilder = minutes > 0 ? strBuilder.append(minutes).append("m").append(" ") : strBuilder; + strBuilder = seconds > 0 ? strBuilder.append(seconds).append("s") : strBuilder; + + return strBuilder.toString(); + + } + + /** + * get monday + *

+ * note: Set the first day of the week to Monday, the default is Sunday + * + * @param date date + * @return get monday + */ + public static Date getMonday(Date date) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + + cal.setFirstDayOfWeek(Calendar.MONDAY); + cal.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY); + + return cal.getTime(); + } + + /** + * get sunday + *

+ * note: Set the first day of the week to Monday, the default is Sunday + * + * @param date date + * @return get sunday + */ + public static Date getSunday(Date date) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + + cal.setFirstDayOfWeek(Calendar.MONDAY); + cal.set(Calendar.DAY_OF_WEEK, Calendar.SUNDAY); + + return cal.getTime(); + } + + /** + * get first day of month + * + * @param date date + * @return first day of month + */ + public static Date getFirstDayOfMonth(Date date) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + cal.set(Calendar.DAY_OF_MONTH, 1); + + return cal.getTime(); + } + + /** + * get some hour of day + * + * @param date date + * @param offsetHour hours + * @return some hour of day + */ + public static Date getSomeHourOfDay(Date date, int offsetHour) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + cal.set(Calendar.HOUR_OF_DAY, cal.get(Calendar.HOUR_OF_DAY) + offsetHour); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTime(); + } + + /** + * get last day of month + * + * @param date date + * @return get last day of month + */ + public static Date getLastDayOfMonth(Date date) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + + cal.add(Calendar.MONTH, 1); + cal.set(Calendar.DAY_OF_MONTH, 1); + cal.add(Calendar.DAY_OF_MONTH, -1); + + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 00:00:00 + * + * @param inputDay date + * @return start day + */ + public static Date getStartOfDay(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 23:59:59 + * + * @param inputDay day + * @return end of day + */ + public static Date getEndOfDay(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.HOUR_OF_DAY, 23); + cal.set(Calendar.MINUTE, 59); + cal.set(Calendar.SECOND, 59); + cal.set(Calendar.MILLISECOND, 999); + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 00:00:00 + * + * @param inputDay day + * @return start of hour + */ + public static Date getStartOfHour(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 23:59:59 + * + * @param inputDay day + * @return end of hour + */ + public static Date getEndOfHour(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.MINUTE, 59); + cal.set(Calendar.SECOND, 59); + cal.set(Calendar.MILLISECOND, 999); + return cal.getTime(); + } + + /** + * get current date + * + * @return current date + */ + public static Date getCurrentDate() { + return DateUtils.parse(DateUtils.getCurrentTime(), + Constants.YYYY_MM_DD_HH_MM_SS); + } + + public static Date addYears(Date date, int amount) { + return add(date, 1, amount); + } + + public static Date addMonths(Date date, int amount) { + return add(date, 2, amount); + } + + public static Date addWeeks(Date date, int amount) { + return add(date, 3, amount); + } + + public static Date addDays(Date date, int amount) { + return add(date, 5, amount); + } + + public static Date addHours(Date date, int amount) { + return add(date, 11, amount); + } + + public static Date addMinutes(Date date, int amount) { + return add(date, 12, amount); + } + + public static Date addSeconds(Date date, int amount) { + return add(date, 13, amount); + } + + public static Date addMilliseconds(Date date, int amount) { + return add(date, 14, amount); + } + + /** + * get date + * + * @param date date + * @param calendarField calendarField + * @param amount amount + * @return date + */ + public static Date add(final Date date, final int calendarField, final int amount) { + if (date == null) { + throw new IllegalArgumentException("The date must not be null"); + } + final Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(calendarField, amount); + return c.getTime(); + } + + /** + * starting from the current time, get how many seconds are left before the target time. + * targetTime = baseTime + intervalSeconds + * + * @param baseTime base time + * @param intervalSeconds a period of time + * @return the number of seconds + */ + public static long getRemainTime(Date baseTime, long intervalSeconds) { + if (baseTime == null) { + return 0; + } + long usedTime = (System.currentTimeMillis() - baseTime.getTime()) / 1000; + return intervalSeconds - usedTime; + } + + /** + * get current time stamp : yyyyMMddHHmmssSSS + * + * @return date string + */ + public static String getCurrentTimeStamp() { + return getCurrentTime(Constants.YYYYMMDDHHMMSSSSS); + } + + /** + * transform date to target timezone date + *

e.g. + *

if input date is 2020-01-01 00:00:00 current timezone is CST + *

targetTimezoneId is MST + *

this method will return 2020-01-01 15:00:00 + */ + public static Date getTimezoneDate(Date date, String targetTimezoneId) { + if (StringUtils.isEmpty(targetTimezoneId)) { + return date; + } + + String dateToString = dateToString(date); + LocalDateTime localDateTime = LocalDateTime.parse(dateToString, DateTimeFormatter.ofPattern(Constants.YYYY_MM_DD_HH_MM_SS)); + ZonedDateTime zonedDateTime = ZonedDateTime.of(localDateTime, TimeZone.getTimeZone(targetTimezoneId).toZoneId()); + return Date.from(zonedDateTime.toInstant()); + } + + /** + * get timezone by timezoneId + */ + public static TimeZone getTimezone(String timezoneId) { + if (StringUtils.isEmpty(timezoneId)) { + return null; + } + return TimeZone.getTimeZone(timezoneId); + } + + /** + * Time unit representing one thousandth of a second + */ + public static class MILLISECONDS { + + public static long toSeconds(long d) { + return d / (C3 / C2); + } + + public static long toMinutes(long d) { + return d / (C4 / C2); + } + + public static long toHours(long d) { + return d / (C5 / C2); + } + + public static long toDays(long d) { + return d / (C6 / C2); + } + + public static long toDurationSeconds(long d) { + return (d % (C4 / C2)) / (C3 / C2); + } + + public static long toDurationMinutes(long d) { + return (d % (C5 / C2)) / (C4 / C2); + } + + public static long toDurationHours(long d) { + return (d % (C6 / C2)) / (C5 / C2); + } + + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java new file mode 100644 index 0000000000..491be64d5d --- /dev/null +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.spi.utils; + +import static org.apache.dolphinscheduler.spi.utils.Constants.COMMON_PROPERTIES_PATH; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PropertyUtils { + private static final Logger logger = LoggerFactory.getLogger(PropertyUtils.class); + + private static final Properties properties = new Properties(); + + private PropertyUtils() { + throw new UnsupportedOperationException("Construct PropertyUtils"); + } + + static { + loadPropertyFile(COMMON_PROPERTIES_PATH); + } + + public static synchronized void loadPropertyFile(String... propertyFiles) { + for (String fileName : propertyFiles) { + try (InputStream fis = PropertyUtils.class.getResourceAsStream(fileName);) { + properties.load(fis); + + } catch (IOException e) { + logger.error(e.getMessage(), e); + System.exit(1); + } + } + + // Override from system properties + System.getProperties().forEach((k, v) -> { + final String key = String.valueOf(k); + logger.info("Overriding property from system property: {}", key); + PropertyUtils.setValue(key, String.valueOf(v)); + }); + } + + /** + * get property value + * + * @param key property name + * @return property value + */ + public static String getString(String key) { + return properties.getProperty(key.trim()); + } + + /** + * get property value with upper case + * + * @param key property name + * @return property value with upper case + */ + public static String getUpperCaseString(String key) { + return properties.getProperty(key.trim()).toUpperCase(); + } + + /** + * get property value + * + * @param key property name + * @param defaultVal default value + * @return property value + */ + public static String getString(String key, String defaultVal) { + String val = properties.getProperty(key.trim()); + return val == null ? defaultVal : val; + } + + /** + * get property value + * + * @param key property name + * @return get property int value , if key == null, then return -1 + */ + public static int getInt(String key) { + return getInt(key, -1); + } + + /** + * @param key key + * @param defaultValue default value + * @return property value + */ + public static int getInt(String key, int defaultValue) { + String value = getString(key); + if (value == null) { + return defaultValue; + } + + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + logger.info(e.getMessage(), e); + } + return defaultValue; + } + + /** + * get property value + * + * @param key property name + * @return property value + */ + public static boolean getBoolean(String key) { + String value = properties.getProperty(key.trim()); + if (null != value) { + return Boolean.parseBoolean(value); + } + + return false; + } + + /** + * get property value + * + * @param key property name + * @param defaultValue default value + * @return property value + */ + public static Boolean getBoolean(String key, boolean defaultValue) { + String value = properties.getProperty(key.trim()); + if (null != value) { + return Boolean.parseBoolean(value); + } + + return defaultValue; + } + + /** + * get property long value + * + * @param key key + * @param defaultVal default value + * @return property value + */ + public static long getLong(String key, long defaultVal) { + String val = getString(key); + return val == null ? defaultVal : Long.parseLong(val); + } + + /** + * @param key key + * @return property value + */ + public static long getLong(String key) { + return getLong(key, -1); + } + + /** + * @param key key + * @param defaultVal default value + * @return property value + */ + public static double getDouble(String key, double defaultVal) { + String val = getString(key); + return val == null ? defaultVal : Double.parseDouble(val); + } + + /** + * get array + * + * @param key property name + * @param splitStr separator + * @return property value through array + */ + public static String[] getArray(String key, String splitStr) { + String value = getString(key); + if (value == null) { + return new String[0]; + } + try { + String[] propertyArray = value.split(splitStr); + return propertyArray; + } catch (NumberFormatException e) { + logger.info(e.getMessage(), e); + } + return new String[0]; + } + + /** + * @param key key + * @param type type + * @param defaultValue default value + * @param T + * @return get enum value + */ + public static > T getEnum(String key, Class type, + T defaultValue) { + String val = getString(key); + return val == null ? defaultValue : Enum.valueOf(type, val); + } + + /** + * get all properties with specified prefix, like: fs. + * + * @param prefix prefix to search + * @return all properties with specified prefix + */ + public static Map getPrefixedProperties(String prefix) { + Map matchedProperties = new HashMap<>(); + for (String propName : properties.stringPropertyNames()) { + if (propName.startsWith(prefix)) { + matchedProperties.put(propName, properties.getProperty(propName)); + } + } + return matchedProperties; + } + + /** + * + */ + public static void setValue(String key, String value) { + properties.setProperty(key, value); + } + + public static Map getPropertiesByPrefix(String prefix) { + if (StringUtils.isEmpty(prefix)) { + return null; + } + Set keys = properties.keySet(); + if (keys.isEmpty()) { + return null; + } + Map propertiesMap = new HashMap<>(); + keys.forEach(k -> { + if (k.toString().contains(prefix)) { + propertiesMap.put(k.toString().replaceFirst(prefix + ".", ""), properties.getProperty((String) k)); + } + }); + return propertiesMap; + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java index fe0306b169..3d6bdfa738 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java @@ -17,25 +17,210 @@ package org.apache.dolphinscheduler.spi.utils; +import java.util.Collection; +import java.util.Iterator; + +/** + * java.lang.String utils class + */ public class StringUtils { + + /** + * The empty String {@code ""}. + */ public static final String EMPTY = ""; + public static final int INDEX_NOT_FOUND = -1; + + private StringUtils() { + throw new UnsupportedOperationException("Construct StringUtils"); + } + + /** + *

Checks if a CharSequence is empty ("") or null.

+ * + * @param cs the CharSequence to check, may be null + * @return {@code true} if the CharSequence is empty or null + */ public static boolean isEmpty(final CharSequence cs) { return cs == null || cs.length() == 0; } + /** + *

Checks if a CharSequence is not empty ("") and not null.

+ * + * @param cs the CharSequence to check, may be null + * @return {@code true} if the CharSequence is not empty and not null + */ public static boolean isNotEmpty(final CharSequence cs) { return !isEmpty(cs); } - public static boolean isBlank(String s) { - if (isEmpty(s)) { + /** + *

Checks if a CharSequence is empty (""), null or whitespace only.

+ * + * @param cs the CharSequence to check, may be null + * @return {@code true} if the CharSequence is null, empty or whitespace only + */ + public static boolean isBlank(final CharSequence cs) { + int strLen; + if (cs == null || (strLen = cs.length()) == 0) { return true; } - return s.trim().length() == 0; + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(cs.charAt(i))) { + return false; + } + } + return true; + } + + /** + *

Checks if a CharSequence is not empty (""), not null and not whitespace only.

+ * + * @param cs the CharSequence to check, may be null + * @return {@code true} if the CharSequence is not empty and not null and not whitespace only + */ + public static boolean isNotBlank(final CharSequence cs) { + return !isBlank(cs); } - public static boolean isNotBlank(String s) { - return !isBlank(s); + /** + *

Replace all strings matching the regular expression \t \n \r with _

+ * + * @param src the String , may be null + * @return the string that has been replaced + */ + public static String replaceNRTtoUnderline(String src) { + return isBlank(src) ? src : src.replaceAll("[\n|\r|\t]", "_"); + } + + /** + *

Removes control characters (char <= 32) from both + * ends of this String, handling {@code null} by returning + * {@code null}.

+ * + * @param str the String to be trimmed, may be null + * @return the trimmed string, {@code null} if null String input + */ + public static String trim(final String str) { + return str == null ? null : str.trim(); + } + + /** + *

Returns either the passed in CharSequence, or if the CharSequence is + * whitespace, empty ("") or {@code null}, the value of {@code defaultStr}.

+ * + * @param the specific kind of CharSequence + * @param str the CharSequence to check, may be null + * @param defaultStr the default CharSequence to return + * if the input is whitespace, empty ("") or {@code null}, may be null + * @return the passed in CharSequence, or the default + */ + public static T defaultIfBlank(final T str, final T defaultStr) { + return isBlank(str) ? defaultStr : str; + } + + /** + *

Compares two String, returning {@code true} if they represent + * equal string, ignoring case.

+ * + * @param str1 the first String, may be null + * @param str2 the second String, may be null + * @return {@code true} if the String are equal, case insensitive, or + * both {@code null} + */ + public static boolean equalsIgnoreCase(String str1, String str2) { + return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2); + } + + public static String substringBefore(final String str, final String separator) { + if (isEmpty(str) || separator == null) { + return str; + } + if (separator.isEmpty()) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return str; + } + return str.substring(0, pos); + } + + public static String substringAfter(final String str, final String separator) { + if (isEmpty(str)) { + return str; + } + if (separator == null) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return EMPTY; + } + return str.substring(pos + separator.length()); + } + + public static long strDigitToLong(String str, long defaultValue) { + if (str == null) { + return defaultValue; + } else { + try { + return Long.parseLong(str); + } catch (NumberFormatException var4) { + return defaultValue; + } + } + } + + /** + *

Joins the elements of the provided Collection into a single String + * containing the provided Collection of elements.

+ * + * @param collection the collection, may be null + * @param separator the separator + * @return a single String + */ + public static String join(Collection collection, String separator) { + return collection == null ? null : join(collection.iterator(), separator); + } + + /** + *

Joins the elements of the provided Iterator into a single String + * containing the provided Iterator of elements.

+ * + * @param iterator the iterator, may be null + * @param separator the separator + * @return a single String + */ + public static String join(Iterator iterator, String separator) { + if (iterator == null) { + return null; + } else if (!iterator.hasNext()) { + return ""; + } else { + Object first = iterator.next(); + if (!iterator.hasNext()) { + return first == null ? "" : first.toString(); + } else { + StringBuilder buf = new StringBuilder(256); + if (first != null) { + buf.append(first); + } + + while (iterator.hasNext()) { + if (separator != null) { + buf.append(separator); + } + + Object obj = iterator.next(); + if (obj != null) { + buf.append(obj); + } + } + return buf.toString(); + } + } } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java index 4dd83953e0..8d3609e378 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java @@ -25,7 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.util.LoggerUtils; import org.apache.dolphinscheduler.plugin.task.util.OSUtils; import org.apache.dolphinscheduler.plugin.task.util.ThreadUtils; import org.apache.dolphinscheduler.spi.task.TaskConstants; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.io.BufferedReader; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTaskExecutor.java index 4240c83801..27c568b226 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTaskExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTaskExecutor.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.plugin.task.api; import org.apache.dolphinscheduler.plugin.task.util.LoggerUtils; import org.apache.dolphinscheduler.spi.task.AbstractTask; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import java.util.List; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractYarnTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractYarnTask.java index 72b9fcec90..d5e3e0d4a8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractYarnTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractYarnTask.java @@ -17,10 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.api; -import org.apache.dolphinscheduler.spi.task.AbstractTask; -import org.apache.dolphinscheduler.spi.task.TaskRequest; - -import org.slf4j.Logger; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; /** * abstract yarn task @@ -47,7 +44,7 @@ public abstract class AbstractYarnTask extends AbstractTaskExecutor { public void handle() throws Exception { try { // SHELL task exit code - TaskResponse response = shellCommandExecutor.run(getCommand()); + TaskResponse response = shellCommandExecutor.run(buildCommand()); setExitStatusCode(response.getExitStatusCode()); setAppIds(response.getAppIds()); setProcessId(response.getProcessId()); @@ -83,7 +80,7 @@ public abstract class AbstractYarnTask extends AbstractTaskExecutor { * @return String * @throws Exception exception */ - protected abstract String getCommand(); + protected abstract String buildCommand(); /** * set main jar name diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java index 331f06a25f..ba24b31f4a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.plugin.task.api; import org.apache.dolphinscheduler.plugin.task.util.OSUtils; import org.apache.dolphinscheduler.spi.task.TaskConstants; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import java.io.File; import java.util.ArrayList; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java index 8e90fd9664..4719dc5e2a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.api; import org.apache.dolphinscheduler.plugin.task.util.OSUtils; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.commons.io.FileUtils; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContextCacheManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContextCacheManager.java index 0203156504..25e05efd23 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContextCacheManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContextCacheManager.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.api; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml index 99ac7cd23a..3feb9ef4de 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml @@ -26,6 +26,7 @@ 4.0.0 dolphinscheduler-task-datax + dolphinscheduler-plugin @@ -39,8 +40,15 @@ ${project.version} - + + com.alibaba + druid + + + + dolphinscheduler-task-datax-${project.version} + - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java new file mode 100644 index 0000000000..fd6ea7a953 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.datax; + +import org.apache.dolphinscheduler.spi.enums.Flag; +import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.ResourceInfo; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * DataX parameter + */ +public class DataxParameters extends AbstractParameters { + + /** + * if custom json config，eg 0, 1 + */ + private int customConfig; + + /** + * if customConfig eq 1 ,then json is usable + */ + private String json; + + /** + * data source type，eg MYSQL, POSTGRES ... + */ + private String dsType; + + /** + * datasource id + */ + private int dataSource; + + /** + * data target type，eg MYSQL, POSTGRES ... + */ + private String dtType; + + /** + * datatarget id + */ + private int dataTarget; + + /** + * sql + */ + private String sql; + + /** + * target table + */ + private String targetTable; + + /** + * Pre Statements + */ + private List preStatements; + + /** + * Post Statements + */ + private List postStatements; + + /** + * speed byte num + */ + private int jobSpeedByte; + + /** + * speed record count + */ + private int jobSpeedRecord; + + /** + * Xms memory + */ + private int xms; + + /** + * Xmx memory + */ + private int xmx; + + public int getCustomConfig() { + return customConfig; + } + + public void setCustomConfig(int customConfig) { + this.customConfig = customConfig; + } + + public String getJson() { + return json; + } + + public void setJson(String json) { + this.json = json; + } + + public String getDsType() { + return dsType; + } + + public void setDsType(String dsType) { + this.dsType = dsType; + } + + public int getDataSource() { + return dataSource; + } + + public void setDataSource(int dataSource) { + this.dataSource = dataSource; + } + + public String getDtType() { + return dtType; + } + + public void setDtType(String dtType) { + this.dtType = dtType; + } + + public int getDataTarget() { + return dataTarget; + } + + public void setDataTarget(int dataTarget) { + this.dataTarget = dataTarget; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public String getTargetTable() { + return targetTable; + } + + public void setTargetTable(String targetTable) { + this.targetTable = targetTable; + } + + public List getPreStatements() { + return preStatements; + } + + public void setPreStatements(List preStatements) { + this.preStatements = preStatements; + } + + public List getPostStatements() { + return postStatements; + } + + public void setPostStatements(List postStatements) { + this.postStatements = postStatements; + } + + public int getJobSpeedByte() { + return jobSpeedByte; + } + + public void setJobSpeedByte(int jobSpeedByte) { + this.jobSpeedByte = jobSpeedByte; + } + + public int getJobSpeedRecord() { + return jobSpeedRecord; + } + + public void setJobSpeedRecord(int jobSpeedRecord) { + this.jobSpeedRecord = jobSpeedRecord; + } + + public int getXms() { + return xms; + } + + public void setXms(int xms) { + this.xms = xms; + } + + public int getXmx() { + return xmx; + } + + public void setXmx(int xmx) { + this.xmx = xmx; + } + + @Override + public boolean checkParameters() { + if (customConfig == Flag.NO.ordinal()) { + return dataSource != 0 + && dataTarget != 0 + && StringUtils.isNotEmpty(sql) + && StringUtils.isNotEmpty(targetTable); + } else { + return StringUtils.isNotEmpty(json); + } + } + + @Override + public List getResourceFilesList() { + return new ArrayList<>(); + } + + @Override + public String toString() { + return "DataxParameters{" + + "customConfig=" + customConfig + + ", json='" + json + '\'' + + ", dsType='" + dsType + '\'' + + ", dataSource=" + dataSource + + ", dtType='" + dtType + '\'' + + ", dataTarget=" + dataTarget + + ", sql='" + sql + '\'' + + ", targetTable='" + targetTable + '\'' + + ", preStatements=" + preStatements + + ", postStatements=" + postStatements + + ", jobSpeedByte=" + jobSpeedByte + + ", jobSpeedRecord=" + jobSpeedRecord + + ", xms=" + xms + + ", xmx=" + xmx + + '}'; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java new file mode 100644 index 0000000000..b48d5e8fff --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java @@ -0,0 +1,570 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.datax; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.RWXR_XR_X; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; + +import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; +import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; +import org.apache.dolphinscheduler.plugin.task.api.TaskResponse; +import org.apache.dolphinscheduler.plugin.task.util.OSUtils; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.enums.Flag; +import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.Property; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.DatasourceUtil; +import org.apache.dolphinscheduler.spi.task.paramparser.ParamUtils; +import org.apache.dolphinscheduler.spi.task.paramparser.ParameterUtils; +import org.apache.dolphinscheduler.spi.task.request.DataxTaskRequest; +import org.apache.dolphinscheduler.spi.utils.CollectionUtils; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.SQLSelect; +import com.alibaba.druid.sql.ast.statement.SQLSelectItem; +import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; +import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; +import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class DataxTask extends AbstractTaskExecutor { + /** + * jvm parameters + */ + public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" "; + /** + * python process(datax only supports version 2.7 by default) + */ + private static final String DATAX_PYTHON = "python2.7"; + private static final Pattern PYTHON_PATH_PATTERN = Pattern.compile("/bin/python[\\d.]*$"); + /** + * datax path + */ + private static final String DATAX_PATH = "${DATAX_HOME}/bin/datax.py"; + /** + * datax channel count + */ + private static final int DATAX_CHANNEL_COUNT = 1; + + /** + * datax parameters + */ + private DataxParameters dataXParameters; + + /** + * shell command executor + */ + private ShellCommandExecutor shellCommandExecutor; + + /** + * taskExecutionContext + */ + private DataxTaskRequest taskExecutionContext; + + /** + * constructor + * + * @param taskExecutionContext taskExecutionContext + */ + public DataxTask(DataxTaskRequest taskExecutionContext) { + super(taskExecutionContext); + this.taskExecutionContext = taskExecutionContext; + + this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle, + taskExecutionContext, logger); + } + + /** + * init DataX config + */ + @Override + public void init() { + logger.info("datax task params {}", taskExecutionContext.getTaskParams()); + dataXParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), DataxParameters.class); + + if (!dataXParameters.checkParameters()) { + throw new RuntimeException("datax task params is not valid"); + } + } + + /** + * run DataX process + * + * @throws Exception if error throws Exception + */ + @Override + public void handle() throws Exception { + try { + // set the name of the current thread + String threadLoggerInfoName = String.format("TaskLogInfo-%s", taskExecutionContext.getTaskAppId()); + Thread.currentThread().setName(threadLoggerInfoName); + + // replace placeholder,and combine local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); + + // run datax procesDataSourceService.s + String jsonFilePath = buildDataxJsonFile(paramsMap); + String shellCommandFilePath = buildShellCommandFile(jsonFilePath, paramsMap); + TaskResponse commandExecuteResult = shellCommandExecutor.run(shellCommandFilePath); + + setExitStatusCode(commandExecuteResult.getExitStatusCode()); + setAppIds(commandExecuteResult.getAppIds()); + setProcessId(commandExecuteResult.getProcessId()); + } catch (Exception e) { + setExitStatusCode(EXIT_CODE_FAILURE); + throw e; + } + } + + /** + * cancel DataX process + * + * @param cancelApplication cancelApplication + * @throws Exception if error throws Exception + */ + @Override + public void cancelApplication(boolean cancelApplication) + throws Exception { + // cancel process + shellCommandExecutor.cancelApplication(); + } + + /** + * build datax configuration file + * + * @return datax json file name + * @throws Exception if error throws Exception + */ + private String buildDataxJsonFile(Map paramsMap) + throws Exception { + // generate json + String fileName = String.format("%s/%s_job.json", + taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId()); + String json; + + Path path = new File(fileName).toPath(); + if (Files.exists(path)) { + return fileName; + } + + if (dataXParameters.getCustomConfig() == Flag.YES.ordinal()) { + json = dataXParameters.getJson().replaceAll("\\r\\n", "\n"); + } else { + ObjectNode job = JSONUtils.createObjectNode(); + job.putArray("content").addAll(buildDataxJobContentJson()); + job.set("setting", buildDataxJobSettingJson()); + + ObjectNode root = JSONUtils.createObjectNode(); + root.set("job", job); + root.set("core", buildDataxCoreJson()); + json = root.toString(); + } + + // replace placeholder + json = ParameterUtils.convertParameterPlaceholders(json, ParamUtils.convert(paramsMap)); + + logger.debug("datax job json : {}", json); + + // create datax json file + FileUtils.writeStringToFile(new File(fileName), json, StandardCharsets.UTF_8); + return fileName; + } + + /** + * build datax job config + * + * @return collection of datax job config JSONObject + * @throws SQLException if error throws SQLException + */ + private List buildDataxJobContentJson() { + + BaseConnectionParam dataSourceCfg = (BaseConnectionParam) DatasourceUtil.buildConnectionParams( + DbType.of(taskExecutionContext.getSourcetype()), + taskExecutionContext.getSourceConnectionParams()); + + BaseConnectionParam dataTargetCfg = (BaseConnectionParam) DatasourceUtil.buildConnectionParams( + DbType.of(taskExecutionContext.getTargetType()), + taskExecutionContext.getTargetConnectionParams()); + + List readerConnArr = new ArrayList<>(); + ObjectNode readerConn = JSONUtils.createObjectNode(); + + ArrayNode sqlArr = readerConn.putArray("querySql"); + for (String sql : new String[]{dataXParameters.getSql()}) { + sqlArr.add(sql); + } + + ArrayNode urlArr = readerConn.putArray("jdbcUrl"); + urlArr.add(DatasourceUtil.getJdbcUrl(DbType.valueOf(dataXParameters.getDsType()), dataSourceCfg)); + + readerConnArr.add(readerConn); + + ObjectNode readerParam = JSONUtils.createObjectNode(); + readerParam.put("username", dataSourceCfg.getUser()); + readerParam.put("password", decodePassword(dataSourceCfg.getPassword())); + readerParam.putArray("connection").addAll(readerConnArr); + + ObjectNode reader = JSONUtils.createObjectNode(); + reader.put("name", DataxUtils.getReaderPluginName(DbType.of(taskExecutionContext.getSourcetype()))); + reader.set("parameter", readerParam); + + List writerConnArr = new ArrayList<>(); + ObjectNode writerConn = JSONUtils.createObjectNode(); + ArrayNode tableArr = writerConn.putArray("table"); + tableArr.add(dataXParameters.getTargetTable()); + + writerConn.put("jdbcUrl", DatasourceUtil.getJdbcUrl(DbType.valueOf(dataXParameters.getDtType()), dataTargetCfg)); + writerConnArr.add(writerConn); + + ObjectNode writerParam = JSONUtils.createObjectNode(); + writerParam.put("username", dataTargetCfg.getUser()); + writerParam.put("password", decodePassword(dataTargetCfg.getPassword())); + + String[] columns = parsingSqlColumnNames(DbType.of(taskExecutionContext.getSourcetype()), + DbType.of(taskExecutionContext.getTargetType()), + dataSourceCfg, dataXParameters.getSql()); + + ArrayNode columnArr = writerParam.putArray("column"); + for (String column : columns) { + columnArr.add(column); + } + writerParam.putArray("connection").addAll(writerConnArr); + + if (CollectionUtils.isNotEmpty(dataXParameters.getPreStatements())) { + ArrayNode preSqlArr = writerParam.putArray("preSql"); + for (String preSql : dataXParameters.getPreStatements()) { + preSqlArr.add(preSql); + } + + } + + if (CollectionUtils.isNotEmpty(dataXParameters.getPostStatements())) { + ArrayNode postSqlArr = writerParam.putArray("postSql"); + for (String postSql : dataXParameters.getPostStatements()) { + postSqlArr.add(postSql); + } + } + + ObjectNode writer = JSONUtils.createObjectNode(); + writer.put("name", DataxUtils.getWriterPluginName(DbType.of(taskExecutionContext.getTargetType()))); + writer.set("parameter", writerParam); + + List contentList = new ArrayList<>(); + ObjectNode content = JSONUtils.createObjectNode(); + content.set("reader", reader); + content.set("writer", writer); + contentList.add(content); + + return contentList; + } + + /** + * build datax setting config + * + * @return datax setting config JSONObject + */ + private ObjectNode buildDataxJobSettingJson() { + + ObjectNode speed = JSONUtils.createObjectNode(); + + speed.put("channel", DATAX_CHANNEL_COUNT); + + if (dataXParameters.getJobSpeedByte() > 0) { + speed.put("byte", dataXParameters.getJobSpeedByte()); + } + + if (dataXParameters.getJobSpeedRecord() > 0) { + speed.put("record", dataXParameters.getJobSpeedRecord()); + } + + ObjectNode errorLimit = JSONUtils.createObjectNode(); + errorLimit.put("record", 0); + errorLimit.put("percentage", 0); + + ObjectNode setting = JSONUtils.createObjectNode(); + setting.set("speed", speed); + setting.set("errorLimit", errorLimit); + + return setting; + } + + private ObjectNode buildDataxCoreJson() { + + ObjectNode speed = JSONUtils.createObjectNode(); + speed.put("channel", DATAX_CHANNEL_COUNT); + + if (dataXParameters.getJobSpeedByte() > 0) { + speed.put("byte", dataXParameters.getJobSpeedByte()); + } + + if (dataXParameters.getJobSpeedRecord() > 0) { + speed.put("record", dataXParameters.getJobSpeedRecord()); + } + + ObjectNode channel = JSONUtils.createObjectNode(); + channel.set("speed", speed); + + ObjectNode transport = JSONUtils.createObjectNode(); + transport.set("channel", channel); + + ObjectNode core = JSONUtils.createObjectNode(); + core.set("transport", transport); + + return core; + } + + /** + * create command + * + * @return shell command file name + * @throws Exception if error throws Exception + */ + private String buildShellCommandFile(String jobConfigFilePath, Map paramsMap) + throws Exception { + // generate scripts + String fileName = String.format("%s/%s_node.%s", + taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId(), + OSUtils.isWindows() ? "bat" : "sh"); + + Path path = new File(fileName).toPath(); + + if (Files.exists(path)) { + return fileName; + } + + // datax python command + StringBuilder sbr = new StringBuilder(); + sbr.append(getPythonCommand()); + sbr.append(" "); + sbr.append(DATAX_PATH); + sbr.append(" "); + sbr.append(loadJvmEnv(dataXParameters)); + sbr.append(jobConfigFilePath); + + // replace placeholder + String dataxCommand = ParameterUtils.convertParameterPlaceholders(sbr.toString(), ParamUtils.convert(paramsMap)); + + logger.debug("raw script : {}", dataxCommand); + + // create shell command file + Set perms = PosixFilePermissions.fromString(RWXR_XR_X); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + + if (OSUtils.isWindows()) { + Files.createFile(path); + } else { + Files.createFile(path, attr); + } + + Files.write(path, dataxCommand.getBytes(), StandardOpenOption.APPEND); + + return fileName; + } + + public String getPythonCommand() { + String pythonHome = System.getenv("PYTHON_HOME"); + return getPythonCommand(pythonHome); + } + + public String getPythonCommand(String pythonHome) { + if (StringUtils.isEmpty(pythonHome)) { + return DATAX_PYTHON; + } + String pythonBinPath = "/bin/" + DATAX_PYTHON; + Matcher matcher = PYTHON_PATH_PATTERN.matcher(pythonHome); + if (matcher.find()) { + return matcher.replaceAll(pythonBinPath); + } + return Paths.get(pythonHome, pythonBinPath).toString(); + } + + public String loadJvmEnv(DataxParameters dataXParameters) { + int xms = Math.max(dataXParameters.getXms(), 1); + int xmx = Math.max(dataXParameters.getXmx(), 1); + return String.format(JVM_PARAM, xms, xmx); + } + + /** + * parsing synchronized column names in SQL statements + * + * @param sourceType the database type of the data source + * @param targetType the database type of the data target + * @param dataSourceCfg the database connection parameters of the data source + * @param sql sql for data synchronization + * @return Keyword converted column names + */ + private String[] parsingSqlColumnNames(DbType sourceType, DbType targetType, BaseConnectionParam dataSourceCfg, String sql) { + String[] columnNames = tryGrammaticalAnalysisSqlColumnNames(sourceType, sql); + + if (columnNames == null || columnNames.length == 0) { + logger.info("try to execute sql analysis query column name"); + columnNames = tryExecuteSqlResolveColumnNames(sourceType, dataSourceCfg, sql); + } + + notNull(columnNames, String.format("parsing sql columns failed : %s", sql)); + + return DataxUtils.convertKeywordsColumns(targetType, columnNames); + } + + /** + * try grammatical parsing column + * + * @param dbType database type + * @param sql sql for data synchronization + * @return column name array + * @throws RuntimeException if error throws RuntimeException + */ + private String[] tryGrammaticalAnalysisSqlColumnNames(DbType dbType, String sql) { + String[] columnNames; + + try { + SQLStatementParser parser = DataxUtils.getSqlStatementParser(dbType, sql); + if (parser == null) { + logger.warn("database driver [{}] is not support grammatical analysis sql", dbType); + return new String[0]; + } + + SQLStatement sqlStatement = parser.parseStatement(); + SQLSelectStatement sqlSelectStatement = (SQLSelectStatement) sqlStatement; + SQLSelect sqlSelect = sqlSelectStatement.getSelect(); + + List selectItemList = null; + if (sqlSelect.getQuery() instanceof SQLSelectQueryBlock) { + SQLSelectQueryBlock block = (SQLSelectQueryBlock) sqlSelect.getQuery(); + selectItemList = block.getSelectList(); + } else if (sqlSelect.getQuery() instanceof SQLUnionQuery) { + SQLUnionQuery unionQuery = (SQLUnionQuery) sqlSelect.getQuery(); + SQLSelectQueryBlock block = (SQLSelectQueryBlock) unionQuery.getRight(); + selectItemList = block.getSelectList(); + } + + notNull(selectItemList, + String.format("select query type [%s] is not support", sqlSelect.getQuery().toString())); + + columnNames = new String[selectItemList.size()]; + for (int i = 0; i < selectItemList.size(); i++) { + SQLSelectItem item = selectItemList.get(i); + + String columnName = null; + + if (item.getAlias() != null) { + columnName = item.getAlias(); + } else if (item.getExpr() != null) { + if (item.getExpr() instanceof SQLPropertyExpr) { + SQLPropertyExpr expr = (SQLPropertyExpr) item.getExpr(); + columnName = expr.getName(); + } else if (item.getExpr() instanceof SQLIdentifierExpr) { + SQLIdentifierExpr expr = (SQLIdentifierExpr) item.getExpr(); + columnName = expr.getName(); + } + } else { + throw new RuntimeException( + String.format("grammatical analysis sql column [ %s ] failed", item.toString())); + } + + if (columnName == null) { + throw new RuntimeException( + String.format("grammatical analysis sql column [ %s ] failed", item.toString())); + } + + columnNames[i] = columnName; + } + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return new String[0]; + } + + return columnNames; + } + + /** + * try to execute sql to resolve column names + * + * @param baseDataSource the database connection parameters + * @param sql sql for data synchronization + * @return column name array + */ + public String[] tryExecuteSqlResolveColumnNames(DbType sourceType, BaseConnectionParam baseDataSource, String sql) { + String[] columnNames; + sql = String.format("SELECT t.* FROM ( %s ) t WHERE 0 = 1", sql); + sql = sql.replace(";", ""); + + try ( + Connection connection = DatasourceUtil.getConnection(sourceType, baseDataSource); + PreparedStatement stmt = connection.prepareStatement(sql); + ResultSet resultSet = stmt.executeQuery()) { + + ResultSetMetaData md = resultSet.getMetaData(); + int num = md.getColumnCount(); + columnNames = new String[num]; + for (int i = 1; i <= num; i++) { + columnNames[i - 1] = md.getColumnName(i); + } + } catch (SQLException e) { + logger.warn(e.getMessage(), e); + return null; + } + + return columnNames; + } + + @Override + public AbstractParameters getParameters() { + return dataXParameters; + } + + private void notNull(Object obj, String message) { + if (obj == null) { + throw new RuntimeException(message); + } + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannel.java new file mode 100644 index 0000000000..fd10af28dd --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannel.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.datax; + +import org.apache.dolphinscheduler.spi.task.AbstractTask; +import org.apache.dolphinscheduler.spi.task.TaskChannel; +import org.apache.dolphinscheduler.spi.task.request.DataxTaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; + +public class DataxTaskChannel implements TaskChannel { + + @Override + public void cancelApplication(boolean status) { + + } + + @Override + public AbstractTask createTask(TaskRequest taskRequest) { + return new DataxTask((DataxTaskRequest) taskRequest); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannelFactory.java new file mode 100644 index 0000000000..bb35b716c0 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannelFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.datax; + +import org.apache.dolphinscheduler.spi.params.base.PluginParams; +import org.apache.dolphinscheduler.spi.task.TaskChannel; +import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; + +import java.util.List; + +public class DataxTaskChannelFactory implements TaskChannelFactory { + + @Override + public String getName() { + return "DATAX"; + } + + @Override + public List getParams() { + return null; + } + + @Override + public TaskChannel create() { + return new DataxTaskChannel(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskPlugin.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskPlugin.java new file mode 100644 index 0000000000..3e2f7ad205 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskPlugin.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.datax; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class DataxTaskPlugin implements DolphinSchedulerPlugin { + + @Override + public Iterable getTaskChannelFactorys() { + return ImmutableList.of(new DataxTaskChannelFactory()); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java new file mode 100644 index 0000000000..c4edc4eab0 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.datax; + +import org.apache.dolphinscheduler.spi.enums.DbType; + +import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; +import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; +import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; +import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerStatementParser; +import com.alibaba.druid.sql.parser.SQLStatementParser; + +public class DataxUtils { + + public static final String DATAX_READER_PLUGIN_MYSQL = "mysqlreader"; + + public static final String DATAX_READER_PLUGIN_POSTGRESQL = "postgresqlreader"; + + public static final String DATAX_READER_PLUGIN_ORACLE = "oraclereader"; + + public static final String DATAX_READER_PLUGIN_SQLSERVER = "sqlserverreader"; + + public static final String DATAX_READER_PLUGIN_CLICKHOUSE = "clickhousereader"; + + public static final String DATAX_WRITER_PLUGIN_MYSQL = "mysqlwriter"; + + public static final String DATAX_WRITER_PLUGIN_POSTGRESQL = "postgresqlwriter"; + + public static final String DATAX_WRITER_PLUGIN_ORACLE = "oraclewriter"; + + public static final String DATAX_WRITER_PLUGIN_SQLSERVER = "sqlserverwriter"; + + public static final String DATAX_WRITER_PLUGIN_CLICKHOUSE = "clickhousewriter"; + + public static String getReaderPluginName(DbType dbType) { + switch (dbType) { + case MYSQL: + return DATAX_READER_PLUGIN_MYSQL; + case POSTGRESQL: + return DATAX_READER_PLUGIN_POSTGRESQL; + case ORACLE: + return DATAX_READER_PLUGIN_ORACLE; + case SQLSERVER: + return DATAX_READER_PLUGIN_SQLSERVER; + case CLICKHOUSE: + return DATAX_READER_PLUGIN_CLICKHOUSE; + default: + return null; + } + } + + public static String getWriterPluginName(DbType dbType) { + switch (dbType) { + case MYSQL: + return DATAX_WRITER_PLUGIN_MYSQL; + case POSTGRESQL: + return DATAX_WRITER_PLUGIN_POSTGRESQL; + case ORACLE: + return DATAX_WRITER_PLUGIN_ORACLE; + case SQLSERVER: + return DATAX_WRITER_PLUGIN_SQLSERVER; + case CLICKHOUSE: + return DATAX_WRITER_PLUGIN_CLICKHOUSE; + default: + return null; + } + } + + public static SQLStatementParser getSqlStatementParser(DbType dbType, String sql) { + switch (dbType) { + case MYSQL: + return new MySqlStatementParser(sql); + case POSTGRESQL: + return new PGSQLStatementParser(sql); + case ORACLE: + return new OracleStatementParser(sql); + case SQLSERVER: + return new SQLServerStatementParser(sql); + default: + return null; + } + } + + public static String[] convertKeywordsColumns(DbType dbType, String[] columns) { + if (columns == null) { + return null; + } + + String[] toColumns = new String[columns.length]; + for (int i = 0; i < columns.length; i++) { + toColumns[i] = doConvertKeywordsColumn(dbType, columns[i]); + } + + return toColumns; + } + + public static String doConvertKeywordsColumn(DbType dbType, String column) { + if (column == null) { + return column; + } + + column = column.trim(); + column = column.replace("`", ""); + column = column.replace("\"", ""); + column = column.replace("'", ""); + + switch (dbType) { + case MYSQL: + return String.format("`%s`", column); + case POSTGRESQL: + return String.format("\"%s\"", column); + case ORACLE: + return String.format("\"%s\"", column); + case SQLSERVER: + return String.format("`%s`", column); + default: + return column; + } + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml index 7510d33f51..11060c6fa4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml @@ -27,21 +27,21 @@ dolphinscheduler-task-flink dolphinscheduler-plugin + - - org.apache.dolphinscheduler - dolphinscheduler-spi - provided - - - org.apache.dolphinscheduler - dolphinscheduler-task-api - ${project.version} - + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + ${project.version} + dolphinscheduler-task-flink-${project.version} - \ No newline at end of file diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTask.java index 0b7c08a682..3bf5b9ac5e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTask.java @@ -18,81 +18,90 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask; -import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.Property; import org.apache.dolphinscheduler.spi.task.ResourceInfo; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.paramparser.ParamUtils; +import org.apache.dolphinscheduler.spi.task.paramparser.ParameterUtils; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.util.ArrayList; import java.util.List; - -import org.slf4j.Logger; +import java.util.Map; public class FlinkTask extends AbstractYarnTask { /** - * flink command - * usage: flink run [OPTIONS] + * flink command + * usage: flink run [OPTIONS] */ private static final String FLINK_COMMAND = "flink"; private static final String FLINK_RUN = "run"; /** - * flink parameters + * flink parameters */ private FlinkParameters flinkParameters; - private String command; - - - private TaskRequest flinkRequest; + /** + * taskExecutionContext + */ + private TaskRequest taskExecutionContext; - public FlinkTask(TaskRequest taskRequest) { - super(taskRequest); - this.flinkRequest = taskRequest; + public FlinkTask(TaskRequest taskExecutionContext) { + super(taskExecutionContext); + this.taskExecutionContext = taskExecutionContext; } @Override - public String getPreScript() { - - // flink run [OPTIONS] - List args = new ArrayList<>(); - - args.add(FLINK_COMMAND); - args.add(FLINK_RUN); - logger.info("flink task args : {}", args); - // other parameters - args.addAll(FlinkArgsUtils.buildArgs(flinkParameters)); - return String.join(" ", args); + public void init() { - } + logger.info("flink task params {}", taskExecutionContext.getTaskParams()); - @Override - public void setCommand(String command) { - this.command = command; + flinkParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), FlinkParameters.class); - } - - @Override - public void init() { + if (flinkParameters == null || !flinkParameters.checkParameters()) { + throw new RuntimeException("flink task params is not valid"); + } + flinkParameters.setQueue(taskExecutionContext.getQueue()); + setMainJarName(); - logger.info("flink task params {}", flinkRequest.getTaskParams()); + if (StringUtils.isNotEmpty(flinkParameters.getMainArgs())) { + String args = flinkParameters.getMainArgs(); - flinkParameters = JSONUtils.parseObject(flinkRequest.getTaskParams(), FlinkParameters.class); + // combining local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); - if (!flinkParameters.checkParameters()) { - throw new TaskException("flink task params is not valid"); + logger.info("param Map : {}", paramsMap); + if (paramsMap != null) { + args = ParameterUtils.convertParameterPlaceholders(args, ParamUtils.convert(paramsMap)); + logger.info("param args : {}", args); + } + flinkParameters.setMainArgs(args); } } /** * create command - * * @return command */ @Override - protected String getCommand() { + protected String buildCommand() { + // flink run [OPTIONS] + List args = new ArrayList<>(); + + args.add(FLINK_COMMAND); + args.add(FLINK_RUN); + logger.info("flink task args : {}", args); + // other parameters + args.addAll(FlinkArgsUtils.buildArgs(flinkParameters)); + + String command = ParameterUtils + .convertParameterPlaceholders(String.join(" ", args), taskExecutionContext.getDefinedParams()); + + logger.info("flink task command : {}", command); return command; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java index 802aed63f2..ceb36c395f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.spi.task.TaskChannel; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; public class FlinkTaskChannel implements TaskChannel { @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml index 0aaea8d4bb..b9c27bdae8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml @@ -40,9 +40,19 @@ ${project.version} + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + dolphinscheduler-task-http-${project.version} - \ No newline at end of file + + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpMethod.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpMethod.java index d949f8a783..571938fc19 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpMethod.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpMethod.java @@ -21,12 +21,12 @@ package org.apache.dolphinscheduler.plugin.task.http; * http method */ public enum HttpMethod { - /** - * 0 get - * 1 post - * 2 head - * 3 put - * 4 delete - */ - GET, POST, HEAD, PUT, DELETE -} \ No newline at end of file + /** + * 0 get + * 1 post + * 2 head + * 3 put + * 4 delete + */ + GET, POST, HEAD, PUT, DELETE +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java index efad7d3fca..0317592b35 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java @@ -53,18 +53,17 @@ public class HttpParameters extends AbstractParameters { */ private String condition; - /** * Connect Timeout * Unit: ms */ - private int connectTimeout ; + private int connectTimeout; /** * Socket Timeout * Unit: ms */ - private int socketTimeout ; + private int socketTimeout; @Override public boolean checkParameters() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java index 83b6a74091..5e0437b698 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.plugin.task.http; /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java index 5aa44d1695..4044b2c57e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java @@ -20,105 +20,103 @@ package org.apache.dolphinscheduler.plugin.task.http; import java.util.Objects; public class HttpProperty { - /** - * key - */ - private String prop; - - /** - * httpParametersType - */ - private HttpParametersType httpParametersType; - - /** - * value - */ - private String value; - - public HttpProperty() { - } - - public HttpProperty(String prop, HttpParametersType httpParametersType, String value) { - this.prop = prop; - this.httpParametersType = httpParametersType; - this.value = value; - } - - /** - * getter method - * - * @return the prop - * @see HttpProperty#prop - */ - public String getProp() { - return prop; - } - - /** - * setter method - * - * @param prop the prop to set - * @see HttpProperty#prop - */ - public void setProp(String prop) { - this.prop = prop; - } - - /** - * getter method - * - * @return the value - * @see HttpProperty#value - */ - public String getValue() { - return value; - } - - /** - * setter method - * - * @param value the value to set - * @see HttpProperty#value - */ - public void setValue(String value) { - this.value = value; - } - - public HttpParametersType getHttpParametersType() { - return httpParametersType; - } - - public void setHttpParametersType(HttpParametersType httpParametersType) { - this.httpParametersType = httpParametersType; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; + /** + * key + */ + private String prop; + + /** + * httpParametersType + */ + private HttpParametersType httpParametersType; + + /** + * value + */ + private String value; + + public HttpProperty() { } - if (o == null || getClass() != o.getClass()) { - return false; + + public HttpProperty(String prop, HttpParametersType httpParametersType, String value) { + this.prop = prop; + this.httpParametersType = httpParametersType; + this.value = value; + } + + /** + * getter method + * + * @return the prop + * @see HttpProperty#prop + */ + public String getProp() { + return prop; + } + + /** + * setter method + * + * @param prop the prop to set + * @see HttpProperty#prop + */ + public void setProp(String prop) { + this.prop = prop; + } + + /** + * getter method + * + * @return the value + * @see HttpProperty#value + */ + public String getValue() { + return value; + } + + /** + * setter method + * + * @param value the value to set + * @see HttpProperty#value + */ + public void setValue(String value) { + this.value = value; } - HttpProperty property = (HttpProperty) o; - return Objects.equals(prop, property.prop) && - Objects.equals(value, property.value); - } - - - @Override - public int hashCode() { - return Objects.hash(prop, value); - } - - @Override - public String toString() { - return "HttpProperty{" + - "prop='" + prop + '\'' + - ", httpParametersType=" + httpParametersType + - ", value='" + value + '\'' + - '}'; - } + public HttpParametersType getHttpParametersType() { + return httpParametersType; + } + + public void setHttpParametersType(HttpParametersType httpParametersType) { + this.httpParametersType = httpParametersType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + HttpProperty property = (HttpProperty) o; + return Objects.equals(prop, property.prop) + && Objects.equals(value, property.value); + } + + @Override + public int hashCode() { + return Objects.hash(prop, value); + } + + @Override + public String toString() { + return "HttpProperty{" + + "prop='" + prop + '\'' + + ", httpParametersType=" + httpParametersType + + ", value='" + value + '\'' + + '}'; + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java index 7b06d50df7..dd6fda5a9a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java @@ -17,20 +17,56 @@ package org.apache.dolphinscheduler.plugin.task.http; +import static org.apache.dolphinscheduler.plugin.task.http.HttpTaskConstants.APPLICATION_JSON; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.TASK_LOG_INFO_FORMAT; + import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; import org.apache.dolphinscheduler.spi.task.AbstractParameters; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.Property; +import org.apache.dolphinscheduler.spi.task.paramparser.ParamUtils; +import org.apache.dolphinscheduler.spi.task.paramparser.ParameterUtils; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; +import org.apache.dolphinscheduler.spi.utils.CollectionUtils; +import org.apache.dolphinscheduler.spi.utils.DateUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.commons.io.Charsets; +import org.apache.http.HttpEntity; +import org.apache.http.ParseException; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.node.ObjectNode; public class HttpTask extends AbstractTaskExecutor { + /** + * output + */ + protected String output; + /** + * http parameters + */ + private HttpParameters httpParameters; /** * taskExecutionContext */ private TaskRequest taskExecutionContext; - private HttpParameters httpParameters; - /** * constructor * @@ -38,6 +74,7 @@ public class HttpTask extends AbstractTaskExecutor { */ public HttpTask(TaskRequest taskExecutionContext) { super(taskExecutionContext); + this.taskExecutionContext = taskExecutionContext; } @Override @@ -53,10 +90,234 @@ public class HttpTask extends AbstractTaskExecutor { @Override public void handle() throws Exception { + String threadLoggerInfoName = String.format(TASK_LOG_INFO_FORMAT, taskExecutionContext.getTaskAppId()); + Thread.currentThread().setName(threadLoggerInfoName); + + long startTime = System.currentTimeMillis(); + String formatTimeStamp = DateUtils.formatTimeStamp(startTime); + String statusCode = null; + String body = null; + + try (CloseableHttpClient client = createHttpClient(); + CloseableHttpResponse response = sendRequest(client)) { + statusCode = String.valueOf(getStatusCode(response)); + body = getResponseBody(response); + exitStatusCode = validResponse(body, statusCode); + long costTime = System.currentTimeMillis() - startTime; + logger.info("startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {} milliseconds, statusCode : {}, body : {}, log : {}", + formatTimeStamp, httpParameters.getUrl(), + httpParameters.getHttpMethod(), costTime, statusCode, body, output); + } catch (Exception e) { + appendMessage(e.toString()); + exitStatusCode = -1; + logger.error("httpUrl[" + httpParameters.getUrl() + "] connection failed：" + output, e); + throw e; + } + + } + + /** + * send request + * + * @param client client + * @return CloseableHttpResponse + * @throws IOException io exception + */ + protected CloseableHttpResponse sendRequest(CloseableHttpClient client) throws IOException { + RequestBuilder builder = createRequestBuilder(); + + // replace placeholder,and combine local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); + + List httpPropertyList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(httpParameters.getHttpParams())) { + for (HttpProperty httpProperty : httpParameters.getHttpParams()) { + String jsonObject = JSONUtils.toJsonString(httpProperty); + String params = ParameterUtils.convertParameterPlaceholders(jsonObject, ParamUtils.convert(paramsMap)); + logger.info("http request params：{}", params); + httpPropertyList.add(JSONUtils.parseObject(params, HttpProperty.class)); + } + } + addRequestParams(builder, httpPropertyList); + String requestUrl = ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParamUtils.convert(paramsMap)); + HttpUriRequest request = builder.setUri(requestUrl).build(); + setHeaders(request, httpPropertyList); + return client.execute(request); + } + + /** + * get response body + * + * @param httpResponse http response + * @return response body + * @throws ParseException parse exception + * @throws IOException io exception + */ + protected String getResponseBody(CloseableHttpResponse httpResponse) throws ParseException, IOException { + if (httpResponse == null) { + return null; + } + HttpEntity entity = httpResponse.getEntity(); + if (entity == null) { + return null; + } + return EntityUtils.toString(entity, StandardCharsets.UTF_8.name()); + } + + /** + * get status code + * + * @param httpResponse http response + * @return status code + */ + protected int getStatusCode(CloseableHttpResponse httpResponse) { + return httpResponse.getStatusLine().getStatusCode(); + } + + /** + * valid response + * + * @param body body + * @param statusCode status code + * @return exit status code + */ + protected int validResponse(String body, String statusCode) { + int exitStatusCode = 0; + switch (httpParameters.getHttpCheckCondition()) { + case BODY_CONTAINS: + if (StringUtils.isEmpty(body) || !body.contains(httpParameters.getCondition())) { + appendMessage(httpParameters.getUrl() + " doesn contain " + + httpParameters.getCondition()); + exitStatusCode = -1; + } + break; + case BODY_NOT_CONTAINS: + if (StringUtils.isEmpty(body) || body.contains(httpParameters.getCondition())) { + appendMessage(httpParameters.getUrl() + " contains " + + httpParameters.getCondition()); + exitStatusCode = -1; + } + break; + case STATUS_CODE_CUSTOM: + if (!statusCode.equals(httpParameters.getCondition())) { + appendMessage(httpParameters.getUrl() + " statuscode: " + statusCode + ", Must be: " + httpParameters.getCondition()); + exitStatusCode = -1; + } + break; + default: + if (!"200".equals(statusCode)) { + appendMessage(httpParameters.getUrl() + " statuscode: " + statusCode + ", Must be: 200"); + exitStatusCode = -1; + } + break; + } + return exitStatusCode; + } + + public String getOutput() { + return output; + } + + /** + * append message + * + * @param message message + */ + protected void appendMessage(String message) { + if (output == null) { + output = ""; + } + if (message != null && !message.trim().isEmpty()) { + output += message; + } + } + + /** + * add request params + * + * @param builder buidler + * @param httpPropertyList http property list + */ + protected void addRequestParams(RequestBuilder builder, List httpPropertyList) { + if (CollectionUtils.isNotEmpty(httpPropertyList)) { + ObjectNode jsonParam = JSONUtils.createObjectNode(); + for (HttpProperty property : httpPropertyList) { + if (property.getHttpParametersType() != null) { + if (property.getHttpParametersType().equals(HttpParametersType.PARAMETER)) { + builder.addParameter(property.getProp(), property.getValue()); + } else if (property.getHttpParametersType().equals(HttpParametersType.BODY)) { + jsonParam.put(property.getProp(), property.getValue()); + } + } + } + StringEntity postingString = new StringEntity(jsonParam.toString(), Charsets.UTF_8); + postingString.setContentEncoding(StandardCharsets.UTF_8.name()); + postingString.setContentType(APPLICATION_JSON); + builder.setEntity(postingString); + } + } + + /** + * set headers + * + * @param request request + * @param httpPropertyList http property list + */ + protected void setHeaders(HttpUriRequest request, List httpPropertyList) { + if (CollectionUtils.isNotEmpty(httpPropertyList)) { + for (HttpProperty property : httpPropertyList) { + if (HttpParametersType.HEADERS.equals(property.getHttpParametersType())) { + request.addHeader(property.getProp(), property.getValue()); + } + } + } + } + + /** + * create http client + * + * @return CloseableHttpClient + */ + protected CloseableHttpClient createHttpClient() { + final RequestConfig requestConfig = requestConfig(); + HttpClientBuilder httpClientBuilder; + httpClientBuilder = HttpClients.custom().setDefaultRequestConfig(requestConfig); + return httpClientBuilder.build(); + } + + /** + * request config + * + * @return RequestConfig + */ + private RequestConfig requestConfig() { + return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout()).setConnectTimeout(httpParameters.getConnectTimeout()).build(); + } + + /** + * create request builder + * + * @return RequestBuilder + */ + protected RequestBuilder createRequestBuilder() { + if (httpParameters.getHttpMethod().equals(HttpMethod.GET)) { + return RequestBuilder.get(); + } else if (httpParameters.getHttpMethod().equals(HttpMethod.POST)) { + return RequestBuilder.post(); + } else if (httpParameters.getHttpMethod().equals(HttpMethod.HEAD)) { + return RequestBuilder.head(); + } else if (httpParameters.getHttpMethod().equals(HttpMethod.PUT)) { + return RequestBuilder.put(); + } else if (httpParameters.getHttpMethod().equals(HttpMethod.DELETE)) { + return RequestBuilder.delete(); + } else { + return null; + } + } @Override public AbstractParameters getParameters() { - return null; + return this.httpParameters; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannel.java index 48aade6919..01484c47cc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannel.java @@ -19,9 +19,10 @@ package org.apache.dolphinscheduler.plugin.task.http; import org.apache.dolphinscheduler.spi.task.AbstractTask; import org.apache.dolphinscheduler.spi.task.TaskChannel; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; public class HttpTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { @@ -31,4 +32,5 @@ public class HttpTaskChannel implements TaskChannel { public AbstractTask createTask(TaskRequest taskRequest) { return new HttpTask(taskRequest); } + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannelFactory.java index 2e3a08d0fd..4791b26a16 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannelFactory.java @@ -24,9 +24,10 @@ import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; import java.util.List; public class HttpTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { - return null; + return "HTTP"; } @Override @@ -36,6 +37,6 @@ public class HttpTaskChannelFactory implements TaskChannelFactory { @Override public TaskChannel create() { - return null; + return new HttpTaskChannel(); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskConstants.java similarity index 83% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskConstants.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskConstants.java index 95fe00b669..60189aceec 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskConstants.java @@ -15,11 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.shell; +package org.apache.dolphinscheduler.plugin.task.http; -public class ShellTaskConstants { +public class HttpTaskConstants { - private ShellTaskConstants() { - - } + public static final String APPLICATION_JSON = "application/json"; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskPlugin.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskPlugin.java index c78a96f872..fa2a4705ca 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskPlugin.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskPlugin.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.plugin.task.http; -import com.google.common.collect.ImmutableList; import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; +import com.google.common.collect.ImmutableList; + public class HttpTaskPlugin implements DolphinSchedulerPlugin { @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml index 6582a31f67..7cf4ffd57b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml @@ -26,6 +26,7 @@ 4.0.0 dolphinscheduler-task-mr + dolphinscheduler-plugin @@ -41,4 +42,8 @@ + + dolphinscheduler-task-mr-${project.version} + + \ No newline at end of file diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceArgsUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceArgsUtils.java new file mode 100644 index 0000000000..84eee6ccd7 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceArgsUtils.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.mr; + +import static org.apache.dolphinscheduler.plugin.task.mr.MapReduceTaskConstants.MR_NAME; +import static org.apache.dolphinscheduler.plugin.task.mr.MapReduceTaskConstants.MR_QUEUE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.D; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.JAR; + +import org.apache.dolphinscheduler.plugin.task.util.ArgsUtils; +import org.apache.dolphinscheduler.spi.task.ResourceInfo; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * mapreduce args utils + */ +public class MapReduceArgsUtils { + + private MapReduceArgsUtils() { + throw new IllegalStateException("Utility class"); + } + + /** + * build args + * + * @param param param + * @return argument list + */ + public static List buildArgs(MapReduceParameters param) { + List args = new ArrayList<>(); + + ResourceInfo mainJar = param.getMainJar(); + if (mainJar != null) { + args.add(JAR); + args.add(mainJar.getRes()); + } + + ProgramType programType = param.getProgramType(); + String mainClass = param.getMainClass(); + if (programType != null && programType != ProgramType.PYTHON && StringUtils.isNotEmpty(mainClass)) { + args.add(mainClass); + } + + String appName = param.getAppName(); + if (StringUtils.isNotEmpty(appName)) { + args.add(String.format("%s%s=%s", D, MR_NAME, ArgsUtils.escape(appName))); + } + + String others = param.getOthers(); + if (StringUtils.isEmpty(others) || !others.contains(MR_QUEUE)) { + String queue = param.getQueue(); + if (StringUtils.isNotEmpty(queue)) { + args.add(String.format("%s%s=%s", D, MR_QUEUE, queue)); + } + } + + // -conf -archives -files -libjars -D + if (StringUtils.isNotEmpty(others)) { + args.add(others); + } + + String mainArgs = param.getMainArgs(); + if (StringUtils.isNotEmpty(mainArgs)) { + args.add(mainArgs); + } + + return args; + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java new file mode 100644 index 0000000000..d099ca6cce --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.mr; + +import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.ResourceInfo; + +import java.util.ArrayList; +import java.util.List; + +/** + * mapreduce parameters + */ +public class MapReduceParameters extends AbstractParameters { + + /** + * major jar + */ + private ResourceInfo mainJar; + + /** + * major class + */ + private String mainClass; + + /** + * arguments + */ + private String mainArgs; + + /** + * other arguments + */ + private String others; + + /** + * app name + */ + private String appName; + + /** + * queue + */ + private String queue; + + /** + * resource list + */ + private List resourceList = new ArrayList<>(); + + /** + * program type + * 0 JAVA,1 SCALA,2 PYTHON + */ + private ProgramType programType; + + public String getMainClass() { + return mainClass; + } + + public void setMainClass(String mainClass) { + this.mainClass = mainClass; + } + + public String getMainArgs() { + return mainArgs; + } + + public void setMainArgs(String mainArgs) { + this.mainArgs = mainArgs; + } + + public String getOthers() { + return others; + } + + public void setOthers(String others) { + this.others = others; + } + + public String getAppName() { + return appName; + } + + public void setAppName(String appName) { + this.appName = appName; + } + + public String getQueue() { + return queue; + } + + public void setQueue(String queue) { + this.queue = queue; + } + + public List getResourceList() { + return this.resourceList; + } + + public void setResourceList(List resourceList) { + this.resourceList = resourceList; + } + + public void setMainJar(ResourceInfo mainJar) { + this.mainJar = mainJar; + } + + public ResourceInfo getMainJar() { + return mainJar; + } + + public ProgramType getProgramType() { + return programType; + } + + public void setProgramType(ProgramType programType) { + this.programType = programType; + } + + @Override + public boolean checkParameters() { + return this.mainJar != null && this.programType != null; + } + + @Override + public List getResourceFilesList() { + if (mainJar != null && !resourceList.contains(mainJar)) { + resourceList.add(mainJar); + } + + return resourceList; + } + + @Override + public String toString() { + return "mainJar= " + mainJar + + "mainClass=" + mainClass + + "mainArgs=" + mainArgs + + "queue=" + queue + + "other mainArgs=" + others + ; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java new file mode 100644 index 0000000000..272a72bf84 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.mr; + +import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask; +import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.Property; +import org.apache.dolphinscheduler.spi.task.ResourceInfo; +import org.apache.dolphinscheduler.spi.task.TaskConstants; +import org.apache.dolphinscheduler.spi.task.paramparser.ParamUtils; +import org.apache.dolphinscheduler.spi.task.paramparser.ParameterUtils; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * mapreduce task + */ +public class MapReduceTask extends AbstractYarnTask { + + /** + * mapreduce command + * usage: hadoop jar [mainClass] [GENERIC_OPTIONS] args... + */ + private static final String MAPREDUCE_COMMAND = TaskConstants.HADOOP; + + /** + * mapreduce parameters + */ + private MapReduceParameters mapreduceParameters; + + /** + * taskExecutionContext + */ + private TaskRequest taskExecutionContext; + + /** + * constructor + * @param taskExecutionContext taskExecutionContext + */ + public MapReduceTask(TaskRequest taskExecutionContext) { + super(taskExecutionContext); + this.taskExecutionContext = taskExecutionContext; + } + + @Override + public void init() { + + logger.info("mapreduce task params {}", taskExecutionContext.getTaskParams()); + + this.mapreduceParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), MapReduceParameters.class); + + // check parameters + if (mapreduceParameters == null || !mapreduceParameters.checkParameters()) { + throw new RuntimeException("mapreduce task params is not valid"); + } + + mapreduceParameters.setQueue(taskExecutionContext.getQueue()); + setMainJarName(); + + // replace placeholder,and combine local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); + + if (paramsMap != null) { + String args = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getMainArgs(), ParamUtils.convert(paramsMap)); + mapreduceParameters.setMainArgs(args); + if (mapreduceParameters.getProgramType() != null && mapreduceParameters.getProgramType() == ProgramType.PYTHON) { + String others = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getOthers(), ParamUtils.convert(paramsMap)); + mapreduceParameters.setOthers(others); + } + } + } + + /** + * build command + * @return command + */ + @Override + protected String buildCommand() { + // hadoop jar [mainClass] [GENERIC_OPTIONS] args... + List args = new ArrayList<>(); + args.add(MAPREDUCE_COMMAND); + + // other parameters + args.addAll(MapReduceArgsUtils.buildArgs(mapreduceParameters)); + + String command = ParameterUtils.convertParameterPlaceholders(String.join(" ", args), + taskExecutionContext.getDefinedParams()); + logger.info("mapreduce task command: {}", command); + + return command; + } + + @Override + protected void setMainJarName() { + // main jar + ResourceInfo mainJar = mapreduceParameters.getMainJar(); + if (mainJar != null) { + int resourceId = mainJar.getId(); + String resourceName; + if (resourceId == 0) { + resourceName = mainJar.getRes(); + } else { + //when update resource maybe has error ,也许也可以交给上层去做控制 需要看资源是否可以抽象为共性 目前来讲我认为是可以的 + resourceName = mainJar.getResourceName().replaceFirst("/", ""); + } + mainJar.setRes(resourceName); + mapreduceParameters.setMainJar(mainJar); + } + } + + @Override + public AbstractParameters getParameters() { + return mapreduceParameters; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java new file mode 100644 index 0000000000..4c721f0859 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.mr; + +import org.apache.dolphinscheduler.spi.task.AbstractTask; +import org.apache.dolphinscheduler.spi.task.TaskChannel; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; + +public class MapReduceTaskChannel implements TaskChannel { + @Override + public void cancelApplication(boolean status) { + + } + + @Override + public AbstractTask createTask(TaskRequest taskRequest) { + return new MapReduceTask(taskRequest); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannelFactory.java new file mode 100644 index 0000000000..165ee1a83e --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannelFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.mr; + +import org.apache.dolphinscheduler.spi.params.base.PluginParams; +import org.apache.dolphinscheduler.spi.task.TaskChannel; +import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; + +import java.util.List; + +public class MapReduceTaskChannelFactory implements TaskChannelFactory { + + @Override + public String getName() { + return "MR"; + } + + @Override + public List getParams() { + return null; + } + + @Override + public TaskChannel create() { + return new MapReduceTaskChannel(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskConstants.java new file mode 100644 index 0000000000..0c438566ac --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskConstants.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.mr; + +public class MapReduceTaskConstants { + + private MapReduceTaskConstants() { + throw new UnsupportedOperationException("Construct Constants"); + } + + /** + * -D mapreduce.job.name=name + */ + public static final String MR_NAME = "mapreduce.job.name"; + + /** + * -D mapreduce.job.queuename=queuename + */ + public static final String MR_QUEUE = "mapreduce.job.queuename"; + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskPlugin.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskPlugin.java new file mode 100644 index 0000000000..4e5657e1cb --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskPlugin.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.mr; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class MapReduceTaskPlugin implements DolphinSchedulerPlugin { + + @Override + public Iterable getTaskChannelFactorys() { + return ImmutableList.of(new MapReduceTaskChannelFactory()); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/ProgramType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/ProgramType.java new file mode 100644 index 0000000000..597a971605 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/ProgramType.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.mr; + +/** + * support program types + */ +public enum ProgramType { + /** + * 0 JAVA,1 SCALA,2 PYTHON + */ + JAVA, + SCALA, + PYTHON +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml index 9c13b8c54a..324ae53e48 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml @@ -26,6 +26,7 @@ 4.0.0 dolphinscheduler-task-procedure + dolphinscheduler-plugin @@ -38,8 +39,10 @@ dolphinscheduler-task-api ${project.version} - + + dolphinscheduler-task-procedure-${project.version} + - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java new file mode 100644 index 0000000000..bfb56c2f5f --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.procedure; + +import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.ResourceInfo; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * procedure parameter + */ +public class ProcedureParameters extends AbstractParameters { + + /** + * data source type，eg MYSQL, POSTGRES, HIVE ... + */ + private String type; + + /** + * data source id + */ + private int datasource; + + /** + * procedure name + */ + private String method; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public int getDatasource() { + return datasource; + } + + public void setDatasource(int datasource) { + this.datasource = datasource; + } + + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + @Override + public boolean checkParameters() { + return datasource != 0 && StringUtils.isNotEmpty(type) && StringUtils.isNotEmpty(method); + } + + @Override + public List getResourceFilesList() { + return new ArrayList<>(); + } + + @Override + public String toString() { + return "ProcessdureParam{" + + "type='" + type + '\'' + + ", datasource=" + datasource + + ", method='" + method + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java new file mode 100644 index 0000000000..ef84036f86 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.procedure; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.EXIT_CODE_SUCCESS; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.TASK_LOG_INFO_FORMAT; + +import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; +import org.apache.dolphinscheduler.spi.enums.DataType; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.enums.TaskTimeoutStrategy; +import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.Direct; +import org.apache.dolphinscheduler.spi.task.Property; +import org.apache.dolphinscheduler.spi.task.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.DatasourceUtil; +import org.apache.dolphinscheduler.spi.task.paramparser.ParamUtils; +import org.apache.dolphinscheduler.spi.task.paramparser.ParameterUtils; +import org.apache.dolphinscheduler.spi.task.request.ProcedureTaskRequest; +import org.apache.dolphinscheduler.spi.utils.CollectionUtils; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * procedure task + */ +public class ProcedureTask extends AbstractTaskExecutor { + + /** + * procedure parameters + */ + private ProcedureParameters procedureParameters; + + /** + * taskExecutionContext + */ + private ProcedureTaskRequest taskExecutionContext; + + /** + * constructor + * + * @param taskExecutionContext taskExecutionContext + */ + public ProcedureTask(ProcedureTaskRequest taskExecutionContext) { + super(taskExecutionContext); + + this.taskExecutionContext = taskExecutionContext; + + logger.info("procedure task params {}", taskExecutionContext.getTaskParams()); + + this.procedureParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), ProcedureParameters.class); + + // check parameters + if (!procedureParameters.checkParameters()) { + throw new RuntimeException("procedure task params is not valid"); + } + } + + @Override + public void handle() throws Exception { + // set the name of the current thread + String threadLoggerInfoName = String.format(TASK_LOG_INFO_FORMAT, taskExecutionContext.getTaskAppId()); + Thread.currentThread().setName(threadLoggerInfoName); + + logger.info("procedure type : {}, datasource : {}, method : {} , localParams : {}", + procedureParameters.getType(), + procedureParameters.getDatasource(), + procedureParameters.getMethod(), + procedureParameters.getLocalParams()); + + Connection connection = null; + CallableStatement stmt = null; + try { + // load class + DbType dbType = DbType.valueOf(procedureParameters.getType()); + // get datasource + ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(DbType.valueOf(procedureParameters.getType()), + taskExecutionContext.getConnectionParams()); + + // get jdbc connection + connection = DatasourceUtil.getConnection(dbType, connectionParam); + + // combining local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); + + // call method + stmt = connection.prepareCall(procedureParameters.getMethod()); + + // set timeout + setTimeout(stmt); + + // outParameterMap + Map outParameterMap = getOutParameterMap(stmt, paramsMap); + + stmt.executeUpdate(); + + // print the output parameters to the log + printOutParameter(stmt, outParameterMap); + + setExitStatusCode(EXIT_CODE_SUCCESS); + } catch (Exception e) { + setExitStatusCode(EXIT_CODE_FAILURE); + logger.error("procedure task error", e); + throw e; + } finally { + close(stmt, connection); + } + } + + /** + * print outParameter + * + * @param stmt CallableStatement + * @param outParameterMap outParameterMap + * @throws SQLException SQLException + */ + private void printOutParameter(CallableStatement stmt, + Map outParameterMap) throws SQLException { + Iterator> iter = outParameterMap.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry en = iter.next(); + + int index = en.getKey(); + Property property = en.getValue(); + String prop = property.getProp(); + DataType dataType = property.getType(); + // get output parameter + getOutputParameter(stmt, index, prop, dataType); + } + } + + /** + * get output parameter + * + * @param stmt CallableStatement + * @param paramsMap paramsMap + * @return outParameterMap + * @throws Exception Exception + */ + private Map getOutParameterMap(CallableStatement stmt, Map paramsMap) throws Exception { + Map outParameterMap = new HashMap<>(); + if (procedureParameters.getLocalParametersMap() == null) { + return outParameterMap; + } + + Collection userDefParamsList = procedureParameters.getLocalParametersMap().values(); + + if (CollectionUtils.isEmpty(userDefParamsList)) { + return outParameterMap; + } + + int index = 1; + for (Property property : userDefParamsList) { + logger.info("localParams : prop : {} , dirct : {} , type : {} , value : {}" + , property.getProp(), + property.getDirect(), + property.getType(), + property.getValue()); + // set parameters + if (property.getDirect().equals(Direct.IN)) { + ParameterUtils.setInParameter(index, stmt, property.getType(), paramsMap.get(property.getProp()).getValue()); + } else if (property.getDirect().equals(Direct.OUT)) { + setOutParameter(index, stmt, property.getType(), paramsMap.get(property.getProp()).getValue()); + property.setValue(paramsMap.get(property.getProp()).getValue()); + outParameterMap.put(index, property); + } + index++; + } + + return outParameterMap; + } + + /** + * set timeout + * + * @param stmt CallableStatement + */ + private void setTimeout(CallableStatement stmt) throws SQLException { + Boolean failed = taskExecutionContext.getTaskTimeoutStrategy() == TaskTimeoutStrategy.FAILED; + Boolean warnFailed = taskExecutionContext.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED; + if (failed || warnFailed) { + stmt.setQueryTimeout(taskExecutionContext.getTaskTimeout()); + } + } + + /** + * close jdbc resource + * + * @param stmt stmt + * @param connection connection + */ + private void close(PreparedStatement stmt, Connection connection) { + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + logger.error("close prepared statement error : {}", e.getMessage(), e); + } + } + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + logger.error("close connection error : {}", e.getMessage(), e); + } + } + } + + /** + * get output parameter + * + * @param stmt stmt + * @param index index + * @param prop prop + * @param dataType dataType + * @throws SQLException SQLException + */ + private void getOutputParameter(CallableStatement stmt, int index, String prop, DataType dataType) throws SQLException { + switch (dataType) { + case VARCHAR: + logger.info("out prameter varchar key : {} , value : {}", prop, stmt.getString(index)); + break; + case INTEGER: + logger.info("out prameter integer key : {} , value : {}", prop, stmt.getInt(index)); + break; + case LONG: + logger.info("out prameter long key : {} , value : {}", prop, stmt.getLong(index)); + break; + case FLOAT: + logger.info("out prameter float key : {} , value : {}", prop, stmt.getFloat(index)); + break; + case DOUBLE: + logger.info("out prameter double key : {} , value : {}", prop, stmt.getDouble(index)); + break; + case DATE: + logger.info("out prameter date key : {} , value : {}", prop, stmt.getDate(index)); + break; + case TIME: + logger.info("out prameter time key : {} , value : {}", prop, stmt.getTime(index)); + break; + case TIMESTAMP: + logger.info("out prameter timestamp key : {} , value : {}", prop, stmt.getTimestamp(index)); + break; + case BOOLEAN: + logger.info("out prameter boolean key : {} , value : {}", prop, stmt.getBoolean(index)); + break; + default: + break; + } + } + + @Override + public AbstractParameters getParameters() { + return procedureParameters; + } + + /** + * set out parameter + * + * @param index index + * @param stmt stmt + * @param dataType dataType + * @param value value + * @throws Exception exception + */ + private void setOutParameter(int index, CallableStatement stmt, DataType dataType, String value) throws Exception { + int sqlType; + switch (dataType) { + case VARCHAR: + sqlType = Types.VARCHAR; + break; + case INTEGER: + case LONG: + sqlType = Types.INTEGER; + break; + case FLOAT: + sqlType = Types.FLOAT; + break; + case DOUBLE: + sqlType = Types.DOUBLE; + break; + case DATE: + sqlType = Types.DATE; + break; + case TIME: + sqlType = Types.TIME; + break; + case TIMESTAMP: + sqlType = Types.TIMESTAMP; + break; + case BOOLEAN: + sqlType = Types.BOOLEAN; + break; + default: + throw new IllegalStateException("Unexpected value: " + dataType); + } + + if (StringUtils.isEmpty(value)) { + stmt.registerOutParameter(index, sqlType); + } else { + stmt.registerOutParameter(index, sqlType, value); + } + } +} \ No newline at end of file diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannel.java new file mode 100644 index 0000000000..d0de0eb574 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannel.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.procedure; + +import org.apache.dolphinscheduler.spi.task.AbstractTask; +import org.apache.dolphinscheduler.spi.task.TaskChannel; +import org.apache.dolphinscheduler.spi.task.request.ProcedureTaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; + +public class ProcedureTaskChannel implements TaskChannel { + + @Override + public void cancelApplication(boolean status) { + + } + + @Override + public AbstractTask createTask(TaskRequest taskRequest) { + return new ProcedureTask((ProcedureTaskRequest) taskRequest); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannelFactory.java new file mode 100644 index 0000000000..2530feb37d --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannelFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.procedure; + +import org.apache.dolphinscheduler.spi.params.base.PluginParams; +import org.apache.dolphinscheduler.spi.task.TaskChannel; +import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; + +import java.util.List; + +public class ProcedureTaskChannelFactory implements TaskChannelFactory { + + @Override + public String getName() { + return "PROCEDURE"; + } + + @Override + public List getParams() { + return null; + } + + @Override + public TaskChannel create() { + return new ProcedureTaskChannel(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskPlugin.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskPlugin.java new file mode 100644 index 0000000000..160bccd236 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskPlugin.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.procedure; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class ProcedureTaskPlugin implements DolphinSchedulerPlugin { + + @Override + public Iterable getTaskChannelFactorys() { + return ImmutableList.of(new ProcedureTaskChannelFactory()); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml index 483cec7c41..e40533149e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml @@ -29,21 +29,19 @@ dolphinscheduler-plugin - - org.apache.dolphinscheduler - dolphinscheduler-spi - provided - - - org.apache.dolphinscheduler - dolphinscheduler-task-api - ${project.version} - + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + ${project.version} + dolphinscheduler-task-python-${project.version} - - \ No newline at end of file diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonCommandExecutor.java index 39da9b0901..d277e14877 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonCommandExecutor.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.python; import org.apache.dolphinscheduler.plugin.task.api.AbstractCommandExecutor; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.dolphinscheduler.spi.utils.StringUtils; import org.apache.commons.io.FileUtils; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java index 75a1fbdf2b..6d77d4555c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java @@ -21,13 +21,10 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskResponse; import org.apache.dolphinscheduler.spi.task.AbstractParameters; -import org.apache.dolphinscheduler.spi.task.AbstractTask; import org.apache.dolphinscheduler.spi.task.TaskConstants; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.slf4j.Logger; - /** * python task */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java index fc9b250047..a3774c5d3e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.python; import org.apache.dolphinscheduler.spi.task.TaskChannel; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; public class PythonTaskChannel implements TaskChannel { @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java index 241988a91f..72ef7bc363 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java @@ -18,15 +18,15 @@ package org.apache.dolphinscheduler.plugin.task.shell; import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; -import org.apache.dolphinscheduler.plugin.task.util.OSUtils; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskResponse; +import org.apache.dolphinscheduler.plugin.task.util.OSUtils; import org.apache.dolphinscheduler.spi.task.AbstractParameters; import org.apache.dolphinscheduler.spi.task.Direct; import org.apache.dolphinscheduler.spi.task.Property; import org.apache.dolphinscheduler.spi.task.TaskConstants; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import java.io.File; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannel.java index fd9a7d211a..73d2c2a885 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannel.java @@ -18,13 +18,9 @@ package org.apache.dolphinscheduler.plugin.task.shell; import org.apache.dolphinscheduler.spi.task.TaskChannel; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; public class ShellTaskChannel implements TaskChannel { - /** - * shell parameters - */ - private ShellParameters shellParameters; @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml index bb583f9e42..500175efa9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml @@ -26,21 +26,21 @@ 4.0.0 dolphinscheduler-task-spark dolphinscheduler-plugin + - - org.apache.dolphinscheduler - dolphinscheduler-spi - provided - - - org.apache.dolphinscheduler - dolphinscheduler-task-api - ${project.version} - + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + ${project.version} + dolphinscheduler-task-spark-${project.version} - - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java index f58c9277be..887a12fa9c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java @@ -19,18 +19,19 @@ package org.apache.dolphinscheduler.plugin.task.spark; import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask; import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.Property; import org.apache.dolphinscheduler.spi.task.ResourceInfo; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.paramparser.ParamUtils; +import org.apache.dolphinscheduler.spi.task.paramparser.ParameterUtils; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import java.util.ArrayList; import java.util.List; - -import org.slf4j.Logger; +import java.util.Map; public class SparkTask extends AbstractYarnTask { - /** * spark1 command * usage: spark-submit [options] [app arguments] @@ -51,19 +52,19 @@ public class SparkTask extends AbstractYarnTask { /** * taskExecutionContext */ - private TaskRequest taskRequest; + private TaskRequest taskExecutionContext; - public SparkTask(TaskRequest taskRequest) { - super(taskRequest); - this.taskRequest = taskRequest; + public SparkTask(TaskRequest taskExecutionContext) { + super(taskExecutionContext); + this.taskExecutionContext = taskExecutionContext; } @Override public void init() { - logger.info("spark task params {}", taskRequest.getTaskParams()); + logger.info("spark task params {}", taskExecutionContext.getTaskParams()); - sparkParameters = JSONUtils.parseObject(taskRequest.getTaskParams(), SparkParameters.class); + sparkParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SparkParameters.class); if (null == sparkParameters) { logger.error("Spark params is null"); @@ -73,12 +74,16 @@ public class SparkTask extends AbstractYarnTask { if (!sparkParameters.checkParameters()) { throw new RuntimeException("spark task params is not valid"); } - sparkParameters.setQueue(taskRequest.getQueue()); + sparkParameters.setQueue(taskExecutionContext.getQueue()); setMainJarName(); } + /** + * create command + * @return command + */ @Override - public String getPreScript() { + protected String buildCommand() { // spark-submit [options] [app arguments] List args = new ArrayList<>(); @@ -93,24 +98,18 @@ public class SparkTask extends AbstractYarnTask { // other parameters args.addAll(SparkArgsUtils.buildArgs(sparkParameters)); - return String.join(" ", args); - } - private String command; + // replace placeholder, and combining local and global parameters + Map paramsMap = ParamUtils.convert(taskExecutionContext,getParameters()); + + String command = null; + + if (null != paramsMap) { + command = ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap)); + } - @Override - public void setCommand(String command) { logger.info("spark task command: {}", command); - this.command = command; - } - /** - * get command - * - * @return command - */ - @Override - protected String getCommand() { return command; } @@ -128,7 +127,7 @@ public class SparkTask extends AbstractYarnTask { if (resourceId == 0) { resourceName = mainJar.getRes(); } else { - //fixme when update resource maybe has error + //when update resource maybe has error ,也许也可以交给上层去做控制 需要看资源是否可以抽象为共性 目前来讲我认为是可以的 resourceName = mainJar.getResourceName().replaceFirst("/", ""); } mainJar.setRes(resourceName); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java index 4d060a82cc..2e9f6b2336 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java @@ -19,11 +19,10 @@ package org.apache.dolphinscheduler.plugin.task.spark; import org.apache.dolphinscheduler.spi.task.AbstractTask; import org.apache.dolphinscheduler.spi.task.TaskChannel; -import org.apache.dolphinscheduler.spi.task.TaskRequest; - -import org.slf4j.Logger; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; public class SparkTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java index 0d67d4aafe..29be5773ea 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java @@ -26,7 +26,7 @@ import java.util.List; public class SparkTaskChannelFactory implements TaskChannelFactory { @Override public String getName() { - return "spark"; + return "SPARK"; } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskPlugin.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskPlugin.java index 20007e7f59..ed35d3d5b5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskPlugin.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskPlugin.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; import com.google.common.collect.ImmutableList; public class SparkTaskPlugin implements DolphinSchedulerPlugin { + @Override public Iterable getTaskChannelFactorys() { return ImmutableList.of(new SparkTaskChannelFactory()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml index fb52e95eed..853be1ac83 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml @@ -26,6 +26,7 @@ 4.0.0 dolphinscheduler-task-sqoop + dolphinscheduler-plugin @@ -41,4 +42,7 @@ + + dolphinscheduler-task-sqoop-${project.version} + \ No newline at end of file diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java new file mode 100644 index 0000000000..e5ed8a1434 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop; + +public final class SqoopConstants { + + private SqoopConstants() { + } + + //sqoop general param + public static final String SQOOP = "sqoop"; + public static final String SQOOP_MR_JOB_NAME = "mapred.job.name"; + public static final String SQOOP_PARALLELISM = "-m"; + public static final String FIELDS_TERMINATED_BY = "--fields-terminated-by"; + public static final String LINES_TERMINATED_BY = "--lines-terminated-by"; + public static final String FIELD_NULL_PLACEHOLDER = "--null-non-string 'NULL' --null-string 'NULL'"; + + //sqoop db + public static final String DB_CONNECT = "--connect"; + public static final String DB_USERNAME = "--username"; + public static final String DB_PWD = "--password"; + public static final String TABLE = "--table"; + public static final String COLUMNS = "--columns"; + public static final String QUERY_WHERE = "where"; + public static final String QUERY = "--query"; + public static final String QUERY_CONDITION = "AND \\$CONDITIONS"; + public static final String QUERY_WITHOUT_CONDITION = "WHERE \\$CONDITIONS"; + public static final String MAP_COLUMN_HIVE = "--map-column-hive"; + public static final String MAP_COLUMN_JAVA = "--map-column-java"; + + + //sqoop hive source + public static final String HCATALOG_DATABASE = "--hcatalog-database"; + public static final String HCATALOG_TABLE = "--hcatalog-table"; + public static final String HCATALOG_PARTITION_KEYS = "--hcatalog-partition-keys"; + public static final String HCATALOG_PARTITION_VALUES = "--hcatalog-partition-values"; + + //sqoop hdfs + public static final String HDFS_EXPORT_DIR = "--export-dir"; + public static final String TARGET_DIR = "--target-dir"; + public static final String COMPRESSION_CODEC = "--compression-codec"; + + //sqoop hive + public static final String HIVE_IMPORT = "--hive-import"; + public static final String HIVE_DATABASE = "--hive-database"; + public static final String HIVE_TABLE = "--hive-table"; + public static final String CREATE_HIVE_TABLE = "--create-hive-table"; + public static final String HIVE_DROP_IMPORT_DELIMS = "--hive-drop-import-delims"; + public static final String HIVE_OVERWRITE = "--hive-overwrite"; + public static final String DELETE_TARGET_DIR = "--delete-target-dir"; + public static final String HIVE_DELIMS_REPLACEMENT = "--hive-delims-replacement"; + public static final String HIVE_PARTITION_KEY = "--hive-partition-key"; + public static final String HIVE_PARTITION_VALUE = "--hive-partition-value"; + + //sqoop update model + public static final String UPDATE_KEY = "--update-key"; + public static final String UPDATE_MODE = "--update-mode"; + + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java new file mode 100644 index 0000000000..407390ef21 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop; + +public enum SqoopJobType { + + CUSTOM(0, "CUSTOM"), + TEMPLATE(1, "TEMPLATE"); + + SqoopJobType(int code, String descp) { + this.code = code; + this.descp = descp; + } + + private final int code; + private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopQueryType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopQueryType.java new file mode 100644 index 0000000000..d610962a1c --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopQueryType.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop; + +public enum SqoopQueryType { + + FORM(0, "SQOOP_QUERY_FORM"), + SQL(1, "SQOOP_QUERY_SQL"); + + private final Integer code; + + private final String desc; + + SqoopQueryType(Integer code, String desc) { + this.code = code; + this.desc = desc; + } + + public Integer getCode() { + return code; + } + + public String getDesc() { + return desc; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java new file mode 100644 index 0000000000..ea6de203f6 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop; + +import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask; +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.SqoopJobGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.Property; +import org.apache.dolphinscheduler.spi.task.paramparser.ParamUtils; +import org.apache.dolphinscheduler.spi.task.paramparser.ParameterUtils; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; + +import java.util.Map; + +/** + * sqoop task extends the shell task + */ +public class SqoopTask extends AbstractYarnTask { + + /** + * sqoop task params + */ + private SqoopParameters sqoopParameters; + + /** + * taskExecutionContext + */ + private final SqoopTaskRequest sqoopTaskExecutionContext; + + public SqoopTask(SqoopTaskRequest taskExecutionContext) { + super(taskExecutionContext); + this.sqoopTaskExecutionContext = taskExecutionContext; + } + + @Override + public void init() { + logger.info("sqoop task params {}", sqoopTaskExecutionContext.getTaskParams()); + sqoopParameters = + JSONUtils.parseObject(sqoopTaskExecutionContext.getTaskParams(), SqoopParameters.class); + //check sqoop task params + if (null == sqoopParameters) { + throw new IllegalArgumentException("Sqoop Task params is null"); + } + + if (!sqoopParameters.checkParameters()) { + throw new IllegalArgumentException("Sqoop Task params check fail"); + } + } + + @Override + protected String buildCommand() { + //get sqoop scripts + SqoopJobGenerator generator = new SqoopJobGenerator(); + String script = generator.generateSqoopJob(sqoopParameters, sqoopTaskExecutionContext); + + // combining local and global parameters + Map paramsMap = ParamUtils.convert(sqoopTaskExecutionContext, getParameters()); + + if (paramsMap != null) { + String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); + logger.info("sqoop script: {}", resultScripts); + return resultScripts; + } + + return null; + } + + @Override + protected void setMainJarName() { + } + + @Override + public AbstractParameters getParameters() { + return sqoopParameters; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannel.java new file mode 100644 index 0000000000..f97b221851 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannel.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop; + +import org.apache.dolphinscheduler.spi.task.AbstractTask; +import org.apache.dolphinscheduler.spi.task.TaskChannel; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; + +public class SqoopTaskChannel implements TaskChannel { + + @Override + public void cancelApplication(boolean status) { + + } + + @Override + public AbstractTask createTask(TaskRequest taskRequest) { + return new SqoopTask((SqoopTaskRequest) taskRequest); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannelFactory.java new file mode 100644 index 0000000000..c89ed00020 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannelFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop; + +import org.apache.dolphinscheduler.spi.params.base.PluginParams; +import org.apache.dolphinscheduler.spi.task.TaskChannel; +import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; + +import java.util.List; + +public class SqoopTaskChannelFactory implements TaskChannelFactory { + + @Override + public String getName() { + return "SQOOP"; + } + + @Override + public List getParams() { + return null; + } + + @Override + public TaskChannel create() { + return new SqoopTaskChannel(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskPlugin.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskPlugin.java new file mode 100644 index 0000000000..e300ee50ee --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskPlugin.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop; + +import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; +import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; + +import com.google.common.collect.ImmutableList; + +public class SqoopTaskPlugin implements DolphinSchedulerPlugin { + + @Override + public Iterable getTaskChannelFactorys() { + return ImmutableList.of(new SqoopTaskChannelFactory()); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java new file mode 100644 index 0000000000..36a99a56b8 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.generator; + +import static org.apache.dolphinscheduler.spi.task.TaskConstants.D; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.EQUAL_SIGN; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SPACE; + +import org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.spi.task.Property; +import org.apache.dolphinscheduler.spi.utils.CollectionUtils; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * common script generator + */ +public class CommonGenerator { + + private static final Logger logger = LoggerFactory.getLogger(CommonGenerator.class); + + public String generate(SqoopParameters sqoopParameters) { + + StringBuilder commonSb = new StringBuilder(); + + try { + //sqoop task model + commonSb.append(SqoopConstants.SQOOP) + .append(SPACE) + .append(sqoopParameters.getModelType()); + + //sqoop map-reduce job name + commonSb.append(SPACE).append(D).append(SPACE) + .append(String.format("%s%s%s", SqoopConstants.SQOOP_MR_JOB_NAME, + EQUAL_SIGN, sqoopParameters.getJobName())); + + //hadoop custom param + List hadoopCustomParams = sqoopParameters.getHadoopCustomParams(); + if (CollectionUtils.isNotEmpty(hadoopCustomParams)) { + for (Property hadoopCustomParam : hadoopCustomParams) { + String hadoopCustomParamStr = String.format("%s%s%s", hadoopCustomParam.getProp(), + EQUAL_SIGN, hadoopCustomParam.getValue()); + + commonSb.append(SPACE).append(D) + .append(SPACE).append(hadoopCustomParamStr); + } + } + + //sqoop custom params + List sqoopAdvancedParams = sqoopParameters.getSqoopAdvancedParams(); + if (CollectionUtils.isNotEmpty(sqoopAdvancedParams)) { + for (Property sqoopAdvancedParam : sqoopAdvancedParams) { + commonSb.append(SPACE).append(sqoopAdvancedParam.getProp()) + .append(SPACE).append(sqoopAdvancedParam.getValue()); + } + } + + //sqoop parallelism + if (sqoopParameters.getConcurrency() > 0) { + commonSb.append(SPACE).append(SqoopConstants.SQOOP_PARALLELISM) + .append(SPACE).append(sqoopParameters.getConcurrency()); + } + } catch (Exception e) { + logger.error(String.format("Sqoop task general param build failed: [%s]", e.getMessage())); + } + + return commonSb.toString(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/ISourceGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/ISourceGenerator.java new file mode 100644 index 0000000000..fcb654293b --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/ISourceGenerator.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.generator; + +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; + +/** + * Source Generator Interface + */ +public interface ISourceGenerator { + + /** + * generate the source script + * + * @param sqoopParameters sqoopParameters + * @param taskExecutionContext taskExecutionContext + * @return source script + */ + String generate(SqoopParameters sqoopParameters, SqoopTaskRequest taskExecutionContext); +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/ITargetGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/ITargetGenerator.java new file mode 100644 index 0000000000..4c2fb4b6da --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/ITargetGenerator.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.generator; + +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; + +/** + * Target Generator Interface + */ +public interface ITargetGenerator { + + /** + * generate the target script + * + * @param sqoopParameters sqoopParameters + * @param taskExecutionContext taskExecutionContext + * @return target script + */ + String generate(SqoopParameters sqoopParameters, SqoopTaskRequest taskExecutionContext); +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java new file mode 100644 index 0000000000..0c23a06918 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.generator; + +import org.apache.dolphinscheduler.plugin.task.sqoop.SqoopJobType; +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.sources.HdfsSourceGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.sources.HiveSourceGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.sources.MysqlSourceGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.targets.HdfsTargetGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.targets.HiveTargetGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.targets.MysqlTargetGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; + +/** + * Sqoop Job Scripts Generator + */ +public class SqoopJobGenerator { + + private static final String MYSQL = "MYSQL"; + private static final String HIVE = "HIVE"; + private static final String HDFS = "HDFS"; + + /** + * target script generator + */ + private ITargetGenerator targetGenerator; + /** + * source script generator + */ + private ISourceGenerator sourceGenerator; + /** + * common script generator + */ + private final CommonGenerator commonGenerator; + + public SqoopJobGenerator() { + commonGenerator = new CommonGenerator(); + } + + private void createSqoopJobGenerator(String sourceType, String targetType) { + sourceGenerator = createSourceGenerator(sourceType); + targetGenerator = createTargetGenerator(targetType); + } + + /** + * get the final sqoop scripts + * + * @param sqoopParameters sqoop params + * @return sqoop scripts + */ + public String generateSqoopJob(SqoopParameters sqoopParameters, SqoopTaskRequest taskExecutionContext) { + + String sqoopScripts = ""; + + if (SqoopJobType.TEMPLATE.getDescp().equals(sqoopParameters.getJobType())) { + createSqoopJobGenerator(sqoopParameters.getSourceType(), sqoopParameters.getTargetType()); + if (sourceGenerator == null || targetGenerator == null) { + throw new RuntimeException("sqoop task source type or target type is null"); + } + + sqoopScripts = String.format("%s%s%s", commonGenerator.generate(sqoopParameters), + sourceGenerator.generate(sqoopParameters, taskExecutionContext), + targetGenerator.generate(sqoopParameters, taskExecutionContext)); + } else if (SqoopJobType.CUSTOM.getDescp().equals(sqoopParameters.getJobType())) { + sqoopScripts = sqoopParameters.getCustomShell().replaceAll("\\r\\n", "\n"); + } + + return sqoopScripts; + } + + /** + * get the source generator + * + * @param sourceType sqoop source type + * @return sqoop source generator + */ + private ISourceGenerator createSourceGenerator(String sourceType) { + switch (sourceType) { + case MYSQL: + return new MysqlSourceGenerator(); + case HIVE: + return new HiveSourceGenerator(); + case HDFS: + return new HdfsSourceGenerator(); + default: + return null; + } + } + + /** + * get the target generator + * + * @param targetType sqoop target type + * @return sqoop target generator + */ + private ITargetGenerator createTargetGenerator(String targetType) { + switch (targetType) { + case MYSQL: + return new MysqlTargetGenerator(); + case HIVE: + return new HiveTargetGenerator(); + case HDFS: + return new HdfsTargetGenerator(); + default: + return null; + } + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java new file mode 100644 index 0000000000..26ab521ca3 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.generator.sources; + +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HDFS_EXPORT_DIR; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SPACE; + +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.ISourceGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.sources.SourceHdfsParameter; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * hdfs source generator + */ +public class HdfsSourceGenerator implements ISourceGenerator { + + private static final Logger logger = LoggerFactory.getLogger(HdfsSourceGenerator.class); + + @Override + public String generate(SqoopParameters sqoopParameters, SqoopTaskRequest taskExecutionContext) { + + StringBuilder hdfsSourceSb = new StringBuilder(); + + try { + SourceHdfsParameter sourceHdfsParameter + = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHdfsParameter.class); + + if (null != sourceHdfsParameter) { + if (StringUtils.isNotEmpty(sourceHdfsParameter.getExportDir())) { + hdfsSourceSb.append(SPACE).append(HDFS_EXPORT_DIR) + .append(SPACE).append(sourceHdfsParameter.getExportDir()); + } else { + throw new IllegalArgumentException("Sqoop hdfs export dir is null"); + } + + } + } catch (Exception e) { + logger.error(String.format("Sqoop hdfs source parmas build failed: [%s]", e.getMessage())); + } + + return hdfsSourceSb.toString(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java new file mode 100644 index 0000000000..69fd835846 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.generator.sources; + +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HCATALOG_DATABASE; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HCATALOG_PARTITION_KEYS; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HCATALOG_PARTITION_VALUES; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HCATALOG_TABLE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SPACE; + +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.ISourceGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.sources.SourceHiveParameter; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * hive source generator + */ +public class HiveSourceGenerator implements ISourceGenerator { + + private static final Logger logger = LoggerFactory.getLogger(HiveSourceGenerator.class); + + @Override + public String generate(SqoopParameters sqoopParameters, SqoopTaskRequest taskExecutionContext) { + + StringBuilder hiveSourceSb = new StringBuilder(); + + try { + SourceHiveParameter sourceHiveParameter + = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHiveParameter.class); + + if (null != sourceHiveParameter) { + if (StringUtils.isNotEmpty(sourceHiveParameter.getHiveDatabase())) { + hiveSourceSb.append(SPACE).append(HCATALOG_DATABASE) + .append(SPACE).append(sourceHiveParameter.getHiveDatabase()); + } + + if (StringUtils.isNotEmpty(sourceHiveParameter.getHiveTable())) { + hiveSourceSb.append(SPACE).append(HCATALOG_TABLE) + .append(SPACE).append(sourceHiveParameter.getHiveTable()); + } + + if (StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionKey()) + && StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionValue())) { + hiveSourceSb.append(SPACE).append(HCATALOG_PARTITION_KEYS) + .append(SPACE).append(sourceHiveParameter.getHivePartitionKey()) + .append(SPACE).append(HCATALOG_PARTITION_VALUES) + .append(SPACE).append(sourceHiveParameter.getHivePartitionValue()); + } + } + } catch (Exception e) { + logger.error(String.format("Sqoop hive source params build failed: [%s]", e.getMessage())); + } + + return hiveSourceSb.toString(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MysqlSourceGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MysqlSourceGenerator.java new file mode 100644 index 0000000000..5c7fb05c03 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MysqlSourceGenerator.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.generator.sources; + +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.COLUMNS; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.DB_CONNECT; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.DB_PWD; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.DB_USERNAME; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.MAP_COLUMN_HIVE; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.MAP_COLUMN_JAVA; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.QUERY; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.QUERY_CONDITION; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.QUERY_WHERE; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.QUERY_WITHOUT_CONDITION; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.TABLE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.COMMA; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_QUOTES; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.EQUAL_SIGN; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SPACE; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; + +import org.apache.dolphinscheduler.plugin.task.sqoop.SqoopQueryType; +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.ISourceGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.sources.SourceMysqlParameter; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.Property; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.DatasourceUtil; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * mysql source generator + */ +public class MysqlSourceGenerator implements ISourceGenerator { + + private static final Logger logger = LoggerFactory.getLogger(MysqlSourceGenerator.class); + + @Override + public String generate(SqoopParameters sqoopParameters, SqoopTaskRequest taskExecutionContext) { + + StringBuilder mysqlSourceSb = new StringBuilder(); + + try { + SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class); + + if (null != sourceMysqlParameter) { + BaseConnectionParam baseDataSource = (BaseConnectionParam) DatasourceUtil.buildConnectionParams( + DbType.of(taskExecutionContext.getSourcetype()), + taskExecutionContext.getSourceConnectionParams()); + + if (null != baseDataSource) { + + mysqlSourceSb.append(SPACE).append(DB_CONNECT) + .append(SPACE).append(DOUBLE_QUOTES) + .append(DatasourceUtil.getJdbcUrl(DbType.MYSQL, baseDataSource)).append(DOUBLE_QUOTES) + .append(SPACE).append(DB_USERNAME) + .append(SPACE).append(baseDataSource.getUser()) + .append(SPACE).append(DB_PWD) + .append(SPACE).append(DOUBLE_QUOTES) + .append(decodePassword(baseDataSource.getPassword())).append(DOUBLE_QUOTES); + + //sqoop table & sql query + if (sourceMysqlParameter.getSrcQueryType() == SqoopQueryType.FORM.getCode()) { + if (StringUtils.isNotEmpty(sourceMysqlParameter.getSrcTable())) { + mysqlSourceSb.append(SPACE).append(TABLE) + .append(SPACE).append(sourceMysqlParameter.getSrcTable()); + } + + if (StringUtils.isNotEmpty(sourceMysqlParameter.getSrcColumns())) { + mysqlSourceSb.append(SPACE).append(COLUMNS) + .append(SPACE).append(sourceMysqlParameter.getSrcColumns()); + } + } else if (sourceMysqlParameter.getSrcQueryType() == SqoopQueryType.SQL.getCode() + && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())) { + + String srcQuery = sourceMysqlParameter.getSrcQuerySql(); + mysqlSourceSb.append(SPACE).append(QUERY) + .append(SPACE).append(DOUBLE_QUOTES).append(srcQuery); + + if (srcQuery.toLowerCase().contains(QUERY_WHERE)) { + mysqlSourceSb.append(SPACE).append(QUERY_CONDITION).append(DOUBLE_QUOTES); + } else { + mysqlSourceSb.append(SPACE).append(QUERY_WITHOUT_CONDITION).append(DOUBLE_QUOTES); + } + } + + //sqoop hive map column + List mapColumnHive = sourceMysqlParameter.getMapColumnHive(); + + if (null != mapColumnHive && !mapColumnHive.isEmpty()) { + StringBuilder columnMap = new StringBuilder(); + for (Property item : mapColumnHive) { + columnMap.append(item.getProp()).append(EQUAL_SIGN).append(item.getValue()).append(COMMA); + } + + if (StringUtils.isNotEmpty(columnMap.toString())) { + mysqlSourceSb.append(SPACE).append(MAP_COLUMN_HIVE) + .append(SPACE).append(columnMap.substring(0, columnMap.length() - 1)); + } + } + + //sqoop map column java + List mapColumnJava = sourceMysqlParameter.getMapColumnJava(); + + if (null != mapColumnJava && !mapColumnJava.isEmpty()) { + StringBuilder columnMap = new StringBuilder(); + for (Property item : mapColumnJava) { + columnMap.append(item.getProp()).append(EQUAL_SIGN).append(item.getValue()).append(COMMA); + } + + if (StringUtils.isNotEmpty(columnMap.toString())) { + mysqlSourceSb.append(SPACE).append(MAP_COLUMN_JAVA) + .append(SPACE).append(columnMap.substring(0, columnMap.length() - 1)); + } + } + } + } + } catch (Exception e) { + logger.error(String.format("Sqoop task mysql source params build failed: [%s]", e.getMessage())); + } + + return mysqlSourceSb.toString(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java new file mode 100644 index 0000000000..b36bdd1b1f --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.generator.targets; + +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.COMPRESSION_CODEC; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.DELETE_TARGET_DIR; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.FIELDS_TERMINATED_BY; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.FIELD_NULL_PLACEHOLDER; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.LINES_TERMINATED_BY; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.TARGET_DIR; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SINGLE_QUOTES; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SPACE; + +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.ITargetGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.targets.TargetHdfsParameter; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * hdfs target generator + */ +public class HdfsTargetGenerator implements ITargetGenerator { + + private static final Logger logger = LoggerFactory.getLogger(HdfsTargetGenerator.class); + + @Override + public String generate(SqoopParameters sqoopParameters, SqoopTaskRequest taskExecutionContext) { + + StringBuilder hdfsTargetSb = new StringBuilder(); + + try { + TargetHdfsParameter targetHdfsParameter = + JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHdfsParameter.class); + + if (null != targetHdfsParameter) { + + if (StringUtils.isNotEmpty(targetHdfsParameter.getTargetPath())) { + hdfsTargetSb.append(SPACE).append(TARGET_DIR) + .append(SPACE).append(targetHdfsParameter.getTargetPath()); + } + + if (StringUtils.isNotEmpty(targetHdfsParameter.getCompressionCodec())) { + hdfsTargetSb.append(SPACE).append(COMPRESSION_CODEC) + .append(SPACE).append(targetHdfsParameter.getCompressionCodec()); + } + + if (StringUtils.isNotEmpty(targetHdfsParameter.getFileType())) { + hdfsTargetSb.append(SPACE).append(targetHdfsParameter.getFileType()); + } + + if (targetHdfsParameter.isDeleteTargetDir()) { + hdfsTargetSb.append(SPACE).append(DELETE_TARGET_DIR); + } + + if (StringUtils.isNotEmpty(targetHdfsParameter.getFieldsTerminated())) { + hdfsTargetSb.append(SPACE).append(FIELDS_TERMINATED_BY) + .append(SPACE).append(SINGLE_QUOTES).append(targetHdfsParameter.getFieldsTerminated()).append(SINGLE_QUOTES); + } + + if (StringUtils.isNotEmpty(targetHdfsParameter.getLinesTerminated())) { + hdfsTargetSb.append(SPACE).append(LINES_TERMINATED_BY) + .append(SPACE).append(SINGLE_QUOTES).append(targetHdfsParameter.getLinesTerminated()).append(SINGLE_QUOTES); + } + + hdfsTargetSb.append(SPACE).append(FIELD_NULL_PLACEHOLDER); + } + } catch (Exception e) { + logger.error(String.format("Sqoop hdfs target params build failed: [%s]", e.getMessage())); + } + + return hdfsTargetSb.toString(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java new file mode 100644 index 0000000000..acab488d1f --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.generator.targets; + +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.CREATE_HIVE_TABLE; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.DELETE_TARGET_DIR; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HIVE_DATABASE; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HIVE_DELIMS_REPLACEMENT; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HIVE_DROP_IMPORT_DELIMS; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HIVE_IMPORT; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HIVE_OVERWRITE; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HIVE_PARTITION_KEY; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HIVE_PARTITION_VALUE; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.HIVE_TABLE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SPACE; + +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.ITargetGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.targets.TargetHiveParameter; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * hive target generator + */ +public class HiveTargetGenerator implements ITargetGenerator { + + private static final Logger logger = LoggerFactory.getLogger(HiveTargetGenerator.class); + + @Override + public String generate(SqoopParameters sqoopParameters, SqoopTaskRequest taskExecutionContext) { + + StringBuilder hiveTargetSb = new StringBuilder(); + + try { + TargetHiveParameter targetHiveParameter = + JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHiveParameter.class); + if (null != targetHiveParameter) { + hiveTargetSb.append(SPACE).append(HIVE_IMPORT); + + if (StringUtils.isNotEmpty(targetHiveParameter.getHiveDatabase()) + && StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())) { + hiveTargetSb.append(SPACE).append(HIVE_DATABASE) + .append(SPACE).append(targetHiveParameter.getHiveDatabase()) + .append(SPACE).append(HIVE_TABLE) + .append(SPACE).append(targetHiveParameter.getHiveTable()); + } + + if (targetHiveParameter.isCreateHiveTable()) { + hiveTargetSb.append(SPACE).append(CREATE_HIVE_TABLE); + } + + if (targetHiveParameter.isDropDelimiter()) { + hiveTargetSb.append(SPACE).append(HIVE_DROP_IMPORT_DELIMS); + } + + if (targetHiveParameter.isHiveOverWrite()) { + hiveTargetSb.append(SPACE).append(HIVE_OVERWRITE) + .append(SPACE).append(DELETE_TARGET_DIR); + } + + if (StringUtils.isNotEmpty(targetHiveParameter.getReplaceDelimiter())) { + hiveTargetSb.append(SPACE).append(HIVE_DELIMS_REPLACEMENT) + .append(SPACE).append(targetHiveParameter.getReplaceDelimiter()); + } + + if (StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionKey()) + && StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionValue())) { + hiveTargetSb.append(SPACE).append(HIVE_PARTITION_KEY) + .append(SPACE).append(targetHiveParameter.getHivePartitionKey()) + .append(SPACE).append(HIVE_PARTITION_VALUE) + .append(SPACE).append(targetHiveParameter.getHivePartitionValue()); + } + + } + } catch (Exception e) { + logger.error(String.format("Sqoop hive target params build failed: [%s]", e.getMessage())); + } + + return hiveTargetSb.toString(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MysqlTargetGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MysqlTargetGenerator.java new file mode 100644 index 0000000000..a79360fbc7 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MysqlTargetGenerator.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.generator.targets; + +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.COLUMNS; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.DB_CONNECT; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.DB_PWD; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.DB_USERNAME; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.FIELDS_TERMINATED_BY; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.LINES_TERMINATED_BY; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.TABLE; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.UPDATE_KEY; +import static org.apache.dolphinscheduler.plugin.task.sqoop.SqoopConstants.UPDATE_MODE; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.DOUBLE_QUOTES; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SINGLE_QUOTES; +import static org.apache.dolphinscheduler.spi.task.TaskConstants.SPACE; +import static org.apache.dolphinscheduler.spi.task.datasource.PasswordUtils.decodePassword; + +import org.apache.dolphinscheduler.plugin.task.sqoop.generator.ITargetGenerator; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; +import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.targets.TargetMysqlParameter; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.spi.task.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.task.datasource.DatasourceUtil; +import org.apache.dolphinscheduler.spi.task.request.SqoopTaskRequest; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * mysql target generator + */ +public class MysqlTargetGenerator implements ITargetGenerator { + + private static final Logger logger = LoggerFactory.getLogger(MysqlTargetGenerator.class); + + @Override + public String generate(SqoopParameters sqoopParameters, SqoopTaskRequest taskExecutionContext) { + + StringBuilder mysqlTargetSb = new StringBuilder(); + + try { + TargetMysqlParameter targetMysqlParameter = + JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class); + + if (null != targetMysqlParameter && targetMysqlParameter.getTargetDatasource() != 0) { + + // get datasource + BaseConnectionParam baseDataSource = (BaseConnectionParam) DatasourceUtil.buildConnectionParams( + DbType.of(taskExecutionContext.getTargetType()), + taskExecutionContext.getTargetConnectionParams()); + + if (null != baseDataSource) { + + mysqlTargetSb.append(SPACE).append(DB_CONNECT) + .append(SPACE).append(DOUBLE_QUOTES) + .append(DatasourceUtil.getJdbcUrl(DbType.MYSQL, baseDataSource)).append(DOUBLE_QUOTES) + .append(SPACE).append(DB_USERNAME) + .append(SPACE).append(baseDataSource.getUser()) + .append(SPACE).append(DB_PWD) + .append(SPACE).append(DOUBLE_QUOTES) + .append(decodePassword(baseDataSource.getPassword())).append(DOUBLE_QUOTES) + .append(SPACE).append(TABLE) + .append(SPACE).append(targetMysqlParameter.getTargetTable()); + + if (StringUtils.isNotEmpty(targetMysqlParameter.getTargetColumns())) { + mysqlTargetSb.append(SPACE).append(COLUMNS) + .append(SPACE).append(targetMysqlParameter.getTargetColumns()); + } + + if (StringUtils.isNotEmpty(targetMysqlParameter.getFieldsTerminated())) { + mysqlTargetSb.append(SPACE).append(FIELDS_TERMINATED_BY); + if (targetMysqlParameter.getFieldsTerminated().contains("'")) { + mysqlTargetSb.append(SPACE).append(targetMysqlParameter.getFieldsTerminated()); + + } else { + mysqlTargetSb.append(SPACE).append(SINGLE_QUOTES).append(targetMysqlParameter.getFieldsTerminated()).append(SINGLE_QUOTES); + } + } + + if (StringUtils.isNotEmpty(targetMysqlParameter.getLinesTerminated())) { + mysqlTargetSb.append(SPACE).append(LINES_TERMINATED_BY); + if (targetMysqlParameter.getLinesTerminated().contains(SINGLE_QUOTES)) { + mysqlTargetSb.append(SPACE).append(targetMysqlParameter.getLinesTerminated()); + } else { + mysqlTargetSb.append(SPACE).append(SINGLE_QUOTES).append(targetMysqlParameter.getLinesTerminated()).append(SINGLE_QUOTES); + } + } + + if (targetMysqlParameter.getIsUpdate() + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey()) + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())) { + mysqlTargetSb.append(SPACE).append(UPDATE_KEY) + .append(SPACE).append(targetMysqlParameter.getTargetUpdateKey()) + .append(SPACE).append(UPDATE_MODE) + .append(SPACE).append(targetMysqlParameter.getTargetUpdateMode()); + } + } + } + } catch (Exception e) { + logger.error(String.format("Sqoop mysql target params build failed: [%s]", e.getMessage())); + } + + return mysqlTargetSb.toString(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java new file mode 100644 index 0000000000..d63734ff81 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.parameter; + +import org.apache.dolphinscheduler.plugin.task.sqoop.SqoopJobType; +import org.apache.dolphinscheduler.spi.task.AbstractParameters; +import org.apache.dolphinscheduler.spi.task.Property; +import org.apache.dolphinscheduler.spi.task.ResourceInfo; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * sqoop parameters + */ +public class SqoopParameters extends AbstractParameters { + + /** + * sqoop job type: + * CUSTOM - custom sqoop job + * TEMPLATE - sqoop template job + */ + private String jobType; + + /** + * customJob eq 1, use customShell + */ + private String customShell; + + /** + * sqoop job name - map-reduce job name + */ + private String jobName; + + /** + * model type + */ + private String modelType; + /** + * concurrency + */ + private int concurrency; + /** + * source type + */ + private String sourceType; + /** + * target type + */ + private String targetType; + /** + * source params + */ + private String sourceParams; + /** + * target params + */ + private String targetParams; + + /** + * hadoop custom param for sqoop job + */ + private List hadoopCustomParams; + + /** + * sqoop advanced param + */ + private List sqoopAdvancedParams; + + public String getModelType() { + return modelType; + } + + public void setModelType(String modelType) { + this.modelType = modelType; + } + + public int getConcurrency() { + return concurrency; + } + + public void setConcurrency(int concurrency) { + this.concurrency = concurrency; + } + + public String getSourceType() { + return sourceType; + } + + public void setSourceType(String sourceType) { + this.sourceType = sourceType; + } + + public String getTargetType() { + return targetType; + } + + public void setTargetType(String targetType) { + this.targetType = targetType; + } + + public String getSourceParams() { + return sourceParams; + } + + public void setSourceParams(String sourceParams) { + this.sourceParams = sourceParams; + } + + public String getTargetParams() { + return targetParams; + } + + public void setTargetParams(String targetParams) { + this.targetParams = targetParams; + } + + public String getJobType() { + return jobType; + } + + public void setJobType(String jobType) { + this.jobType = jobType; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getCustomShell() { + return customShell; + } + + public void setCustomShell(String customShell) { + this.customShell = customShell; + } + + public List getHadoopCustomParams() { + return hadoopCustomParams; + } + + public void setHadoopCustomParams(List hadoopCustomParams) { + this.hadoopCustomParams = hadoopCustomParams; + } + + public List getSqoopAdvancedParams() { + return sqoopAdvancedParams; + } + + public void setSqoopAdvancedParams(List sqoopAdvancedParams) { + this.sqoopAdvancedParams = sqoopAdvancedParams; + } + + @Override + public boolean checkParameters() { + + boolean sqoopParamsCheck = false; + + if (StringUtils.isEmpty(jobType)) { + return sqoopParamsCheck; + } + + if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) { + sqoopParamsCheck = StringUtils.isEmpty(customShell) + && StringUtils.isNotEmpty(modelType) + && StringUtils.isNotEmpty(jobName) + && concurrency != 0 + && StringUtils.isNotEmpty(sourceType) + && StringUtils.isNotEmpty(targetType) + && StringUtils.isNotEmpty(sourceParams) + && StringUtils.isNotEmpty(targetParams); + } else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) { + sqoopParamsCheck = StringUtils.isNotEmpty(customShell) + && StringUtils.isEmpty(jobName); + } + + return sqoopParamsCheck; + } + + @Override + public List getResourceFilesList() { + return new ArrayList<>(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceHdfsParameter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceHdfsParameter.java new file mode 100644 index 0000000000..cc84851a9f --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceHdfsParameter.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.parameter.sources; + +/** + * source hdfs parameter + */ +public class SourceHdfsParameter { + + /** + * export dir + */ + private String exportDir; + + public String getExportDir() { + return exportDir; + } + + public void setExportDir(String exportDir) { + this.exportDir = exportDir; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceHiveParameter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceHiveParameter.java new file mode 100644 index 0000000000..0a7cb836fa --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceHiveParameter.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.parameter.sources; + +/** + * source hive parameter + */ +public class SourceHiveParameter { + + /** + * hive database + */ + private String hiveDatabase; + /** + * hive table + */ + private String hiveTable; + /** + * hive partition key + */ + private String hivePartitionKey; + /** + * hive partition value + */ + private String hivePartitionValue; + + public String getHiveDatabase() { + return hiveDatabase; + } + + public void setHiveDatabase(String hiveDatabase) { + this.hiveDatabase = hiveDatabase; + } + + public String getHiveTable() { + return hiveTable; + } + + public void setHiveTable(String hiveTable) { + this.hiveTable = hiveTable; + } + + public String getHivePartitionKey() { + return hivePartitionKey; + } + + public void setHivePartitionKey(String hivePartitionKey) { + this.hivePartitionKey = hivePartitionKey; + } + + public String getHivePartitionValue() { + return hivePartitionValue; + } + + public void setHivePartitionValue(String hivePartitionValue) { + this.hivePartitionValue = hivePartitionValue; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceMysqlParameter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceMysqlParameter.java new file mode 100644 index 0000000000..07c5c27591 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/sources/SourceMysqlParameter.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.parameter.sources; + +import org.apache.dolphinscheduler.spi.task.Property; + +import java.util.List; + +/** + * source mysql parameter + */ +public class SourceMysqlParameter { + + /** + * src datasource + */ + private int srcDatasource; + /** + * src table + */ + private String srcTable; + /** + * src query type + */ + private int srcQueryType; + /** + * src query sql + */ + private String srcQuerySql; + /** + * src column type + */ + private int srcColumnType; + /** + * src columns + */ + private String srcColumns; + /** + * src condition list + */ + private List srcConditionList; + /** + * map column hive + */ + private List mapColumnHive; + /** + * map column java + */ + private List mapColumnJava; + + public int getSrcDatasource() { + return srcDatasource; + } + + public void setSrcDatasource(int srcDatasource) { + this.srcDatasource = srcDatasource; + } + + public String getSrcTable() { + return srcTable; + } + + public void setSrcTable(String srcTable) { + this.srcTable = srcTable; + } + + public int getSrcQueryType() { + return srcQueryType; + } + + public void setSrcQueryType(int srcQueryType) { + this.srcQueryType = srcQueryType; + } + + public String getSrcQuerySql() { + return srcQuerySql; + } + + public void setSrcQuerySql(String srcQuerySql) { + this.srcQuerySql = srcQuerySql; + } + + public int getSrcColumnType() { + return srcColumnType; + } + + public void setSrcColumnType(int srcColumnType) { + this.srcColumnType = srcColumnType; + } + + public String getSrcColumns() { + return srcColumns; + } + + public void setSrcColumns(String srcColumns) { + this.srcColumns = srcColumns; + } + + public List getSrcConditionList() { + return srcConditionList; + } + + public void setSrcConditionList(List srcConditionList) { + this.srcConditionList = srcConditionList; + } + + public List getMapColumnHive() { + return mapColumnHive; + } + + public void setMapColumnHive(List mapColumnHive) { + this.mapColumnHive = mapColumnHive; + } + + public List getMapColumnJava() { + return mapColumnJava; + } + + public void setMapColumnJava(List mapColumnJava) { + this.mapColumnJava = mapColumnJava; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetHdfsParameter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetHdfsParameter.java new file mode 100644 index 0000000000..d63460b38a --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetHdfsParameter.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.parameter.targets; + +/** + * target hdfs parameter + */ +public class TargetHdfsParameter { + + /** + * target dir + */ + private String targetPath; + /** + * delete target dir + */ + private boolean deleteTargetDir; + /** + * file type + */ + private String fileType; + /** + * compression codec + */ + private String compressionCodec; + /** + * fields terminated + */ + private String fieldsTerminated; + /** + * lines terminated + */ + private String linesTerminated; + + public String getTargetPath() { + return targetPath; + } + + public void setTargetPath(String targetPath) { + this.targetPath = targetPath; + } + + public boolean isDeleteTargetDir() { + return deleteTargetDir; + } + + public void setDeleteTargetDir(boolean deleteTargetDir) { + this.deleteTargetDir = deleteTargetDir; + } + + public String getFileType() { + return fileType; + } + + public void setFileType(String fileType) { + this.fileType = fileType; + } + + public String getCompressionCodec() { + return compressionCodec; + } + + public void setCompressionCodec(String compressionCodec) { + this.compressionCodec = compressionCodec; + } + + public String getFieldsTerminated() { + return fieldsTerminated; + } + + public void setFieldsTerminated(String fieldsTerminated) { + this.fieldsTerminated = fieldsTerminated; + } + + public String getLinesTerminated() { + return linesTerminated; + } + + public void setLinesTerminated(String linesTerminated) { + this.linesTerminated = linesTerminated; + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetHiveParameter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetHiveParameter.java new file mode 100644 index 0000000000..7358de7546 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetHiveParameter.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.parameter.targets; + +/** + * target hive parameter + */ +public class TargetHiveParameter { + + /** + * hive database + */ + private String hiveDatabase; + /** + * hive table + */ + private String hiveTable; + /** + * create hive table + */ + private boolean createHiveTable; + /** + * drop delimiter + */ + private boolean dropDelimiter; + /** + * hive overwrite + */ + private boolean hiveOverWrite; + /** + * replace delimiter + */ + private String replaceDelimiter; + /** + * hive partition key + */ + private String hivePartitionKey; + /** + * hive partition value + */ + private String hivePartitionValue; + + public String getHiveDatabase() { + return hiveDatabase; + } + + public void setHiveDatabase(String hiveDatabase) { + this.hiveDatabase = hiveDatabase; + } + + public String getHiveTable() { + return hiveTable; + } + + public void setHiveTable(String hiveTable) { + this.hiveTable = hiveTable; + } + + public boolean isCreateHiveTable() { + return createHiveTable; + } + + public void setCreateHiveTable(boolean createHiveTable) { + this.createHiveTable = createHiveTable; + } + + public boolean isDropDelimiter() { + return dropDelimiter; + } + + public void setDropDelimiter(boolean dropDelimiter) { + this.dropDelimiter = dropDelimiter; + } + + public boolean isHiveOverWrite() { + return hiveOverWrite; + } + + public void setHiveOverWrite(boolean hiveOverWrite) { + this.hiveOverWrite = hiveOverWrite; + } + + public String getReplaceDelimiter() { + return replaceDelimiter; + } + + public void setReplaceDelimiter(String replaceDelimiter) { + this.replaceDelimiter = replaceDelimiter; + } + + public String getHivePartitionKey() { + return hivePartitionKey; + } + + public void setHivePartitionKey(String hivePartitionKey) { + this.hivePartitionKey = hivePartitionKey; + } + + public String getHivePartitionValue() { + return hivePartitionValue; + } + + public void setHivePartitionValue(String hivePartitionValue) { + this.hivePartitionValue = hivePartitionValue; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetMysqlParameter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetMysqlParameter.java new file mode 100644 index 0000000000..bb97f65050 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/targets/TargetMysqlParameter.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop.parameter.targets; + +/** + * target mysql parameter + */ +public class TargetMysqlParameter { + + /** + * target datasource + */ + private int targetDatasource; + /** + * target table + */ + private String targetTable; + /** + * target columns + */ + private String targetColumns; + /** + * fields terminated + */ + private String fieldsTerminated; + /** + * lines terminated + */ + private String linesTerminated; + /** + * pre query + */ + private String preQuery; + /** + * is update + */ + private boolean isUpdate; + /** + * target update key + */ + private String targetUpdateKey; + /** + * target update mode + */ + private String targetUpdateMode; + + public int getTargetDatasource() { + return targetDatasource; + } + + public void setTargetDatasource(int targetDatasource) { + this.targetDatasource = targetDatasource; + } + + public String getTargetTable() { + return targetTable; + } + + public void setTargetTable(String targetTable) { + this.targetTable = targetTable; + } + + public String getTargetColumns() { + return targetColumns; + } + + public void setTargetColumns(String targetColumns) { + this.targetColumns = targetColumns; + } + + public String getFieldsTerminated() { + return fieldsTerminated; + } + + public void setFieldsTerminated(String fieldsTerminated) { + this.fieldsTerminated = fieldsTerminated; + } + + public String getLinesTerminated() { + return linesTerminated; + } + + public void setLinesTerminated(String linesTerminated) { + this.linesTerminated = linesTerminated; + } + + public String getPreQuery() { + return preQuery; + } + + public void setPreQuery(String preQuery) { + this.preQuery = preQuery; + } + + public boolean getIsUpdate() { + return isUpdate; + } + + public void setUpdate(boolean update) { + isUpdate = update; + } + + public String getTargetUpdateKey() { + return targetUpdateKey; + } + + public void setTargetUpdateKey(String targetUpdateKey) { + this.targetUpdateKey = targetUpdateKey; + } + + public String getTargetUpdateMode() { + return targetUpdateMode; + } + + public void setTargetUpdateMode(String targetUpdateMode) { + this.targetUpdateMode = targetUpdateMode; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTask.java index a2fac90dec..60d494971a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTask.java @@ -20,9 +20,8 @@ package org.apache.dolphinscheduler.plugin.task.tis; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; import org.apache.dolphinscheduler.spi.task.AbstractParameters; -import org.apache.dolphinscheduler.spi.task.AbstractTask; import org.apache.dolphinscheduler.spi.task.TaskConstants; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; @@ -177,6 +176,7 @@ public class TISTask extends AbstractTaskExecutor { // WebSocket connection closed } + @Override public void onTextFrame(String payload, boolean finalFragment, int rsv) { ExecLog execLog = JSONUtils.parseObject(payload, ExecLog.class); logger.info(execLog.getMsg()); @@ -221,6 +221,7 @@ public class TISTask extends AbstractTaskExecutor { private static class BizResult extends AjaxResult { private TriggerBuildResult bizresult; + @Override public TriggerBuildResult getBizresult() { return this.bizresult; } @@ -233,6 +234,7 @@ public class TISTask extends AbstractTaskExecutor { private static class StatusResult extends AjaxResult { private Map bizresult; + @Override public Map getBizresult() { return this.bizresult; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannel.java index bef3be9aeb..6ae45d6218 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannel.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.plugin.task.tis; import org.apache.dolphinscheduler.spi.task.AbstractTask; import org.apache.dolphinscheduler.spi.task.TaskChannel; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; public class TISTaskChannel implements TaskChannel { @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/test/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/test/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.java index 6a0cc9be7a..1789a071de 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/test/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/test/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.java @@ -22,7 +22,7 @@ import static com.github.dreamhead.moco.MocoJsonRunner.jsonHttpServer; import static com.github.dreamhead.moco.Runner.running; import org.apache.dolphinscheduler.spi.task.ExecutionStatus; -import org.apache.dolphinscheduler.spi.task.TaskRequest; +import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.commons.io.IOUtils; diff --git a/dolphinscheduler-task-plugin/pom.xml b/dolphinscheduler-task-plugin/pom.xml index e8ba5aaf71..052f8f015c 100644 --- a/dolphinscheduler-task-plugin/pom.xml +++ b/dolphinscheduler-task-plugin/pom.xml @@ -29,19 +29,17 @@ pom - dolphinscheduler-task-shell dolphinscheduler-task-api + dolphinscheduler-task-shell + dolphinscheduler-task-datax dolphinscheduler-task-flink + dolphinscheduler-task-http + dolphinscheduler-task-mr dolphinscheduler-task-python dolphinscheduler-task-spark - dolphinscheduler-task-http dolphinscheduler-task-sql dolphinscheduler-task-sqoop - dolphinscheduler-task-datax - dolphinscheduler-task-mr dolphinscheduler-task-procedure - dolphinscheduler-task-tis - diff --git a/pom.xml b/pom.xml index fbe2079289..f91b43c532 100644 --- a/pom.xml +++ b/pom.xml @@ -682,7 +682,7 @@ ca.vanzyl.maven.plugins provisio-maven-plugin - 1.0.7 + 1.0.4 true