From aa0974fd1f759e96430d3f1b8dac291d6ea7388c Mon Sep 17 00:00:00 2001 From: Yelli Date: Wed, 2 Dec 2020 15:50:50 +0800 Subject: [PATCH] split sqoop import hive database and table (#4141) --- .../server/worker/task/sqoop/SqoopConstants.java | 1 + .../task/sqoop/generator/targets/HiveTargetGenerator.java | 7 ++++--- .../server/worker/task/sqoop/SqoopTaskTest.java | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopConstants.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopConstants.java index c19a1e0fff..772807b4d2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopConstants.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopConstants.java @@ -57,6 +57,7 @@ public final class SqoopConstants { //sqoop hive public static final String HIVE_IMPORT = "--hive-import"; + public static final String HIVE_DATABASE = "--hive-database"; public static final String HIVE_TABLE = "--hive-table"; public static final String CREATE_HIVE_TABLE = "--create-hive-table"; public static final String HIVE_DROP_IMPORT_DELIMS = "--hive-drop-import-delims"; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java index 69a95d008c..769fc62f25 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java @@ -49,9 +49,10 @@ public class HiveTargetGenerator implements ITargetGenerator { if (StringUtils.isNotEmpty(targetHiveParameter.getHiveDatabase()) && StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())) { - hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_TABLE) - .append(Constants.SPACE).append(String.format("%s.%s", targetHiveParameter.getHiveDatabase(), - targetHiveParameter.getHiveTable())); + hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_DATABASE) + .append(Constants.SPACE).append(targetHiveParameter.getHiveDatabase()) + .append(Constants.SPACE).append(SqoopConstants.HIVE_TABLE) + .append(Constants.SPACE).append(targetHiveParameter.getHiveTable()); } if (targetHiveParameter.isCreateHiveTable()) { diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java index fcf15d6e61..0a4124044a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java @@ -144,7 +144,7 @@ public class SqoopTaskTest { String mysqlToHiveScript = generator.generateSqoopJob(mysqlToHiveParams, mysqlTaskExecutionContext); String mysqlToHiveExpected = "sqoop import -D mapred.job.name=sqoop_import -m 1 --connect \"jdbc:mysql://192.168.0.111:3306/test\" --username kylo --password \"123456\" " - + "--query \"SELECT * FROM person_2 WHERE \\$CONDITIONS\" --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 " + + "--query \"SELECT * FROM person_2 WHERE \\$CONDITIONS\" --map-column-java id=Integer --hive-import --hive-database stg --hive-table person_internal_2 " + "--create-hive-table --hive-overwrite --delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16"; Assert.assertEquals(mysqlToHiveExpected, mysqlToHiveScript);