diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DataxUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DataxUtils.java index 930098919b..9eba4f977e 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DataxUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DataxUtils.java @@ -36,6 +36,8 @@ public class DataxUtils { public static final String DATAX_READER_PLUGIN_SQLSERVER = "sqlserverreader"; + public static final String DATAX_READER_PLUGIN_CLICKHOUSE = "clickhousereader"; + public static final String DATAX_WRITER_PLUGIN_MYSQL = "mysqlwriter"; public static final String DATAX_WRITER_PLUGIN_POSTGRESQL = "postgresqlwriter"; @@ -44,6 +46,8 @@ public class DataxUtils { public static final String DATAX_WRITER_PLUGIN_SQLSERVER = "sqlserverwriter"; + public static final String DATAX_WRITER_PLUGIN_CLICKHOUSE = "clickhousewriter"; + public static String getReaderPluginName(DbType dbType) { switch (dbType) { case MYSQL: @@ -54,6 +58,8 @@ public class DataxUtils { return DATAX_READER_PLUGIN_ORACLE; case SQLSERVER: return DATAX_READER_PLUGIN_SQLSERVER; + case CLICKHOUSE: + return DATAX_READER_PLUGIN_CLICKHOUSE; default: return null; } @@ -69,6 +75,8 @@ public class DataxUtils { return DATAX_WRITER_PLUGIN_ORACLE; case SQLSERVER: return DATAX_WRITER_PLUGIN_SQLSERVER; + case CLICKHOUSE: + return DATAX_WRITER_PLUGIN_CLICKHOUSE; default: return null; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 8f0b20d17f..8f80963f89 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -310,8 +310,8 @@ public class DataxTask extends AbstractTask { List contentList = new ArrayList<>(); ObjectNode content = JSONUtils.createObjectNode(); - content.put("reader", reader.toString()); - content.put("writer", writer.toString()); + content.set("reader", reader); + content.set("writer", writer); contentList.add(content); return contentList; @@ -341,8 +341,8 @@ public class DataxTask extends AbstractTask { errorLimit.put("percentage", 0); ObjectNode setting = JSONUtils.createObjectNode(); - setting.put("speed", speed.toString()); - setting.put("errorLimit", errorLimit.toString()); + setting.set("speed", speed); + setting.set("errorLimit", errorLimit); return setting; } @@ -462,7 +462,10 @@ public class DataxTask extends AbstractTask { try { SQLStatementParser parser = DataxUtils.getSqlStatementParser(dbType, sql); - notNull(parser, String.format("database driver [%s] is not support", dbType.toString())); + if (parser == null) { + logger.warn("database driver [{}] is not support grammatical analysis sql", dbType); + return new String[0]; + } SQLStatement sqlStatement = parser.parseStatement(); SQLSelectStatement sqlSelectStatement = (SQLSelectStatement) sqlStatement; @@ -511,7 +514,7 @@ public class DataxTask extends AbstractTask { } } catch (Exception e) { logger.warn(e.getMessage(), e); - return null; + return new String[0]; } return columnNames; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/DataxUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/DataxUtilsTest.java index 2720bb8a28..3c20ba793b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/DataxUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/DataxUtilsTest.java @@ -14,17 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.utils; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.dolphinscheduler.common.enums.DbType; + +import org.junit.Test; + import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerStatementParser; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.junit.Assert; -import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * DataxUtils Tester. @@ -42,6 +45,7 @@ public class DataxUtilsTest { assertEquals(DataxUtils.DATAX_READER_PLUGIN_POSTGRESQL, DataxUtils.getReaderPluginName(DbType.POSTGRESQL)); assertEquals(DataxUtils.DATAX_READER_PLUGIN_SQLSERVER, DataxUtils.getReaderPluginName(DbType.SQLSERVER)); assertEquals(DataxUtils.DATAX_READER_PLUGIN_ORACLE, DataxUtils.getReaderPluginName(DbType.ORACLE)); + assertEquals(DataxUtils.DATAX_READER_PLUGIN_CLICKHOUSE, DataxUtils.getReaderPluginName(DbType.CLICKHOUSE)); assertTrue(DataxUtils.getReaderPluginName(DbType.DB2) == null); } @@ -56,6 +60,7 @@ public class DataxUtilsTest { assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_POSTGRESQL, DataxUtils.getWriterPluginName(DbType.POSTGRESQL)); assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_SQLSERVER, DataxUtils.getWriterPluginName(DbType.SQLSERVER)); assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_ORACLE, DataxUtils.getWriterPluginName(DbType.ORACLE)); + assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_CLICKHOUSE, DataxUtils.getWriterPluginName(DbType.CLICKHOUSE)); assertTrue(DataxUtils.getWriterPluginName(DbType.DB2) == null); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index 8d03c1460a..f52146bd46 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -43,6 +43,7 @@ import java.util.UUID; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; @@ -304,6 +305,7 @@ public class DataxTaskTest { * Method: buildDataxJsonFile() */ @Test + @Ignore("method not found") public void testBuildDataxJsonFile() throws Exception { @@ -322,6 +324,7 @@ public class DataxTaskTest { * Method: buildDataxJsonFile() */ @Test + @Ignore("method not found") public void testBuildDataxJsonFile0() throws Exception { try { @@ -348,17 +351,29 @@ public class DataxTaskTest { Assert.assertNotNull(contentList); ObjectNode content = contentList.get(0); - JsonNode reader = JSONUtils.parseObject(content.path("reader").asText()); + JsonNode reader = JSONUtils.parseObject(content.path("reader").toString()); Assert.assertNotNull(reader); + Assert.assertEquals("{\"name\":\"mysqlreader\",\"parameter\":{\"username\":\"root\"," + + "\"password\":\"123456\",\"connection\":[{\"querySql\":[\"select 1 as test from dual\"]," + + "\"jdbcUrl\":[\"jdbc:mysql://127.0.0.1:3306/test?allowLoadLocalInfile=false" + + "&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\"]}]}}", + reader.toString()); String readerPluginName = reader.path("name").asText(); Assert.assertEquals(DataxUtils.DATAX_READER_PLUGIN_MYSQL, readerPluginName); - JsonNode writer = JSONUtils.parseObject(content.path("writer").asText()); + JsonNode writer = JSONUtils.parseObject(content.path("writer").toString()); Assert.assertNotNull(writer); + Assert.assertEquals("{\"name\":\"mysqlwriter\",\"parameter\":{\"username\":\"root\"," + + "\"password\":\"123456\",\"column\":[\"`test`\"],\"connection\":[{\"table\":[\"test\"]," + + "\"jdbcUrl\":\"jdbc:mysql://127.0.0.1:3306/test?allowLoadLocalInfile=false&" + + "autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\"}]," + + "\"preSql\":[\"delete from test\"],\"postSql\":[\"delete from test\"]}}", + writer.toString()); String writerPluginName = writer.path("name").asText(); Assert.assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_MYSQL, writerPluginName); + } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -375,8 +390,8 @@ public class DataxTaskTest { method.setAccessible(true); JsonNode setting = (JsonNode) method.invoke(dataxTask, null); Assert.assertNotNull(setting); - Assert.assertNotNull(setting.get("speed")); - Assert.assertNotNull(setting.get("errorLimit")); + Assert.assertEquals("{\"channel\":1,\"record\":1000}", setting.get("speed").toString()); + Assert.assertEquals("{\"record\":0,\"percentage\":0}", setting.get("errorLimit").toString()); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -403,6 +418,7 @@ public class DataxTaskTest { * Method: buildShellCommandFile(String jobConfigFilePath) */ @Test + @Ignore("method not found") public void testBuildShellCommandFile() throws Exception { try { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue index 0049ac0036..a469b42b0a 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue @@ -33,7 +33,7 @@ diff --git a/pom.xml b/pom.xml index 660da7ba0e..3d892ddc0c 100644 --- a/pom.xml +++ b/pom.xml @@ -952,7 +952,7 @@ **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java - + **/server/worker/task/datax/DataxTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/worker/task/shell/ShellTaskTest.java