Browse Source

[feature-11408] add hive option (#11493)

3.1.0-release
fuchanghai 2 years ago committed by GitHub
parent
commit
d8d5d392a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
  2. 11
      dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java

2
dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java

@ -565,7 +565,7 @@ public class DataxTask extends AbstractTaskExecutor {
int num = md.getColumnCount(); int num = md.getColumnCount();
columnNames = new String[num]; columnNames = new String[num];
for (int i = 1; i <= num; i++) { for (int i = 1; i <= num; i++) {
columnNames[i - 1] = md.getColumnName(i); columnNames[i - 1] = md.getColumnName(i).replaceAll("t.","");
} }
} catch (SQLException | ExecutionException e) { } catch (SQLException | ExecutionException e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);

11
dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java

@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.task.datax;
import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.enums.DbType;
import com.alibaba.druid.sql.dialect.clickhouse.parser.ClickhouseStatementParser; import com.alibaba.druid.sql.dialect.clickhouse.parser.ClickhouseStatementParser;
import com.alibaba.druid.sql.dialect.hive.parser.HiveStatementParser;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser;
import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser;
@ -38,6 +39,8 @@ public class DataxUtils {
public static final String DATAX_READER_PLUGIN_CLICKHOUSE = "clickhousereader"; public static final String DATAX_READER_PLUGIN_CLICKHOUSE = "clickhousereader";
public static final String DATAX_READER_PLUGIN_HIVE = "rdbmsreader";
public static final String DATAX_WRITER_PLUGIN_MYSQL = "mysqlwriter"; public static final String DATAX_WRITER_PLUGIN_MYSQL = "mysqlwriter";
public static final String DATAX_WRITER_PLUGIN_POSTGRESQL = "postgresqlwriter"; public static final String DATAX_WRITER_PLUGIN_POSTGRESQL = "postgresqlwriter";
@ -48,6 +51,8 @@ public class DataxUtils {
public static final String DATAX_WRITER_PLUGIN_CLICKHOUSE = "clickhousewriter"; public static final String DATAX_WRITER_PLUGIN_CLICKHOUSE = "clickhousewriter";
public static final String DATAX_WRITER_PLUGIN_HIVE = "rdbmswriter";
public static String getReaderPluginName(DbType dbType) { public static String getReaderPluginName(DbType dbType) {
switch (dbType) { switch (dbType) {
case MYSQL: case MYSQL:
@ -60,6 +65,8 @@ public class DataxUtils {
return DATAX_READER_PLUGIN_SQLSERVER; return DATAX_READER_PLUGIN_SQLSERVER;
case CLICKHOUSE: case CLICKHOUSE:
return DATAX_READER_PLUGIN_CLICKHOUSE; return DATAX_READER_PLUGIN_CLICKHOUSE;
case HIVE:
return DATAX_READER_PLUGIN_HIVE;
default: default:
return null; return null;
} }
@ -77,6 +84,8 @@ public class DataxUtils {
return DATAX_WRITER_PLUGIN_SQLSERVER; return DATAX_WRITER_PLUGIN_SQLSERVER;
case CLICKHOUSE: case CLICKHOUSE:
return DATAX_WRITER_PLUGIN_CLICKHOUSE; return DATAX_WRITER_PLUGIN_CLICKHOUSE;
case HIVE:
return DATAX_WRITER_PLUGIN_HIVE;
default: default:
return null; return null;
} }
@ -94,6 +103,8 @@ public class DataxUtils {
return new SQLServerStatementParser(sql); return new SQLServerStatementParser(sql);
case CLICKHOUSE: case CLICKHOUSE:
return new ClickhouseStatementParser(sql); return new ClickhouseStatementParser(sql);
case HIVE:
return new HiveStatementParser(sql);
default: default:
return null; return null;
} }

Loading…
Cancel
Save