[Improvement][Task] DataX Node doesn't Support Clickhouse as datasource (#5180) (#5243)

* [Improvement][Module Name] DataX Node doesn't Support Clickhouse as the datasource (#5180)

* add ut

* add ut

* resolve code smell

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DataxUtils.java

@@ -36,6 +36,8 @@ public class DataxUtils {
 
     public static final String DATAX_READER_PLUGIN_SQLSERVER = "sqlserverreader";
 
+    public static final String DATAX_READER_PLUGIN_CLICKHOUSE = "clickhousereader";
+
     public static final String DATAX_WRITER_PLUGIN_MYSQL = "mysqlwriter";
 
     public static final String DATAX_WRITER_PLUGIN_POSTGRESQL = "postgresqlwriter";
@@ -44,6 +46,8 @@ public class DataxUtils {
 
     public static final String DATAX_WRITER_PLUGIN_SQLSERVER = "sqlserverwriter";
 
+    public static final String DATAX_WRITER_PLUGIN_CLICKHOUSE = "clickhousewriter";
+
     public static String getReaderPluginName(DbType dbType) {
         switch (dbType) {
             case MYSQL:
@@ -54,6 +58,8 @@ public class DataxUtils {
                 return DATAX_READER_PLUGIN_ORACLE;
             case SQLSERVER:
                 return DATAX_READER_PLUGIN_SQLSERVER;
+            case CLICKHOUSE:
+                return DATAX_READER_PLUGIN_CLICKHOUSE;
             default:
                 return null;
         }
@@ -69,6 +75,8 @@ public class DataxUtils {
                 return DATAX_WRITER_PLUGIN_ORACLE;
             case SQLSERVER:
                 return DATAX_WRITER_PLUGIN_SQLSERVER;
+            case CLICKHOUSE:
+                return DATAX_WRITER_PLUGIN_CLICKHOUSE;
             default:
                 return null;
         }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java

@@ -310,8 +310,8 @@ public class DataxTask extends AbstractTask {
 
         List<ObjectNode> contentList = new ArrayList<>();
         ObjectNode content = JSONUtils.createObjectNode();
-        content.put("reader", reader.toString());
+        content.set("reader", reader);
-        content.put("writer", writer.toString());
+        content.set("writer", writer);
         contentList.add(content);
 
         return contentList;
@@ -341,8 +341,8 @@ public class DataxTask extends AbstractTask {
         errorLimit.put("percentage", 0);
 
         ObjectNode setting = JSONUtils.createObjectNode();
-        setting.put("speed", speed.toString());
+        setting.set("speed", speed);
-        setting.put("errorLimit", errorLimit.toString());
+        setting.set("errorLimit", errorLimit);
 
         return setting;
     }
@@ -462,7 +462,10 @@ public class DataxTask extends AbstractTask {
 
         try {
             SQLStatementParser parser = DataxUtils.getSqlStatementParser(dbType, sql);
-            notNull(parser, String.format("database driver [%s] is not support", dbType.toString()));
+            if (parser == null) {
+                logger.warn("database driver [{}] is not support grammatical analysis sql", dbType);
+                return new String[0];
+            }
 
             SQLStatement sqlStatement = parser.parseStatement();
             SQLSelectStatement sqlSelectStatement = (SQLSelectStatement) sqlStatement;
@@ -511,7 +514,7 @@ public class DataxTask extends AbstractTask {
             }
         } catch (Exception e) {
             logger.warn(e.getMessage(), e);
-            return null;
+            return new String[0];
         }
 
         return columnNames;

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/DataxUtilsTest.java

@@ -14,17 +14,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.server.utils;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.dolphinscheduler.common.enums.DbType;
+
+import org.junit.Test;
+
 import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
 import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser;
 import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser;
 import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerStatementParser;
-import org.apache.dolphinscheduler.common.enums.DbType;
-import org.junit.Assert;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 /**
  * DataxUtils Tester.
@@ -42,6 +45,7 @@ public class DataxUtilsTest {
         assertEquals(DataxUtils.DATAX_READER_PLUGIN_POSTGRESQL, DataxUtils.getReaderPluginName(DbType.POSTGRESQL));
         assertEquals(DataxUtils.DATAX_READER_PLUGIN_SQLSERVER, DataxUtils.getReaderPluginName(DbType.SQLSERVER));
         assertEquals(DataxUtils.DATAX_READER_PLUGIN_ORACLE, DataxUtils.getReaderPluginName(DbType.ORACLE));
+        assertEquals(DataxUtils.DATAX_READER_PLUGIN_CLICKHOUSE, DataxUtils.getReaderPluginName(DbType.CLICKHOUSE));
         assertTrue(DataxUtils.getReaderPluginName(DbType.DB2) == null);
     }
 
@@ -56,6 +60,7 @@ public class DataxUtilsTest {
         assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_POSTGRESQL, DataxUtils.getWriterPluginName(DbType.POSTGRESQL));
         assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_SQLSERVER, DataxUtils.getWriterPluginName(DbType.SQLSERVER));
         assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_ORACLE, DataxUtils.getWriterPluginName(DbType.ORACLE));
+        assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_CLICKHOUSE, DataxUtils.getWriterPluginName(DbType.CLICKHOUSE));
         assertTrue(DataxUtils.getWriterPluginName(DbType.DB2) == null);
     }
 

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java

@@ -43,6 +43,7 @@ import java.util.UUID;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.powermock.api.mockito.PowerMockito;
@@ -304,6 +305,7 @@ public class DataxTaskTest {
      * Method: buildDataxJsonFile()
      */
     @Test
+    @Ignore("method not found")
     public void testBuildDataxJsonFile()
             throws Exception {
 
@@ -322,6 +324,7 @@ public class DataxTaskTest {
      * Method: buildDataxJsonFile()
      */
     @Test
+    @Ignore("method not found")
     public void testBuildDataxJsonFile0()
             throws Exception {
         try {
@@ -348,17 +351,29 @@ public class DataxTaskTest {
             Assert.assertNotNull(contentList);
 
             ObjectNode content = contentList.get(0);
-            JsonNode reader = JSONUtils.parseObject(content.path("reader").asText());
+            JsonNode reader = JSONUtils.parseObject(content.path("reader").toString());
             Assert.assertNotNull(reader);
+            Assert.assertEquals("{\"name\":\"mysqlreader\",\"parameter\":{\"username\":\"root\","
+                            + "\"password\":\"123456\",\"connection\":[{\"querySql\":[\"select 1 as test from dual\"],"
+                            + "\"jdbcUrl\":[\"jdbc:mysql://127.0.0.1:3306/test?allowLoadLocalInfile=false"
+                            + "&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\"]}]}}",
+                    reader.toString());
 
             String readerPluginName = reader.path("name").asText();
             Assert.assertEquals(DataxUtils.DATAX_READER_PLUGIN_MYSQL, readerPluginName);
 
-            JsonNode writer = JSONUtils.parseObject(content.path("writer").asText());
+            JsonNode writer = JSONUtils.parseObject(content.path("writer").toString());
             Assert.assertNotNull(writer);
+            Assert.assertEquals("{\"name\":\"mysqlwriter\",\"parameter\":{\"username\":\"root\","
+                            + "\"password\":\"123456\",\"column\":[\"`test`\"],\"connection\":[{\"table\":[\"test\"],"
+                            + "\"jdbcUrl\":\"jdbc:mysql://127.0.0.1:3306/test?allowLoadLocalInfile=false&"
+                            + "autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\"}],"
+                            + "\"preSql\":[\"delete from test\"],\"postSql\":[\"delete from test\"]}}",
+                    writer.toString());
 
             String writerPluginName = writer.path("name").asText();
             Assert.assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_MYSQL, writerPluginName);
+
         } catch (Exception e) {
             Assert.fail(e.getMessage());
         }
@@ -375,8 +390,8 @@ public class DataxTaskTest {
             method.setAccessible(true);
             JsonNode setting = (JsonNode) method.invoke(dataxTask, null);
             Assert.assertNotNull(setting);
-            Assert.assertNotNull(setting.get("speed"));
+            Assert.assertEquals("{\"channel\":1,\"record\":1000}", setting.get("speed").toString());
-            Assert.assertNotNull(setting.get("errorLimit"));
+            Assert.assertEquals("{\"record\":0,\"percentage\":0}", setting.get("errorLimit").toString());
         } catch (Exception e) {
             Assert.fail(e.getMessage());
         }
@@ -403,6 +418,7 @@ public class DataxTaskTest {
      * Method: buildShellCommandFile(String jobConfigFilePath)
      */
     @Test
+    @Ignore("method not found")
     public void testBuildShellCommandFile()
             throws Exception {
         try {

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue

@@ -33,7 +33,7 @@
           <m-datasource
             ref="refDs"
             @on-dsData="_onDsData"
-            :supportType="['MYSQL','POSTGRESQL', 'ORACLE', 'SQLSERVER']"
+            :supportType="['MYSQL','POSTGRESQL', 'ORACLE', 'SQLSERVER', 'CLICKHOUSE']"
             :data="{ type:dsType,datasource:datasource }">
           </m-datasource>
         </div>

pom.xml

@@ -952,7 +952,7 @@
                         <include>**/server/worker/task/spark/SparkTaskTest.java</include>
                         <include>**/server/worker/task/EnvFileTest.java</include>
                         <include>**/server/worker/task/spark/SparkTaskTest.java</include>
-                        <!--<include>**/server/worker/task/datax/DataxTaskTest.java</include>-->
+                        <include>**/server/worker/task/datax/DataxTaskTest.java</include>
                         <!--<include>**/server/worker/task/http/HttpTaskTest.java</include>-->
                         <include>**/server/worker/task/sqoop/SqoopTaskTest.java</include>
                         <include>**/server/worker/task/shell/ShellTaskTest.java</include>