From 034cc20da95a01abeea3b42bebdcd217d0c919f0 Mon Sep 17 00:00:00 2001 From: Baoqi Date: Thu, 11 Apr 2019 19:57:05 +0800 Subject: [PATCH] add ClickHouse jdbc support --- .../api/service/DataSourceService.java | 12 ++- .../cn/escheduler/api/utils/Constants.java | 3 + escheduler-common/pom.xml | 15 ++++ .../java/cn/escheduler/common/Constants.java | 8 +- .../cn/escheduler/common/enums/DbType.java | 3 +- .../common/job/db/ClickHouseDataSource.java | 75 +++++++++++++++++++ .../common/job/db/DataSourceFactory.java | 2 + .../task/processdure/ProcedureTask.java | 6 ++ .../server/worker/task/sql/SqlTask.java | 3 + .../dag/_source/formModel/tasks/procedure.vue | 2 +- .../pages/list/_source/createDataSource.vue | 1 + .../src/js/conf/home/store/dag/state.js | 5 ++ .../js/conf/home/store/datasource/actions.js | 4 +- pom.xml | 6 ++ 14 files changed, 138 insertions(+), 7 deletions(-) create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/job/db/ClickHouseDataSource.java diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index d5371e5f0a..087c427eba 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -217,6 +217,9 @@ public class DataSourceService extends BaseService{ case POSTGRESQL: separator = "&"; break; + case CLICKHOUSE: + separator = "&"; + break; default: separator = "&"; break; @@ -367,6 +370,10 @@ public class DataSourceService extends BaseService{ datasource = JSONObject.parseObject(parameter, SparkDataSource.class); Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); break; + case CLICKHOUSE: + datasource = JSONObject.parseObject(parameter, ClickHouseDataSource.class); + Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); + break; default: break; } @@ -428,7 +435,7 @@ public class DataSourceService extends BaseService{ String address = buildAddress(type, host, port); String jdbcUrl = address + "/" + database; String separator = ""; - if (Constants.MYSQL.equals(type.name()) || Constants.POSTGRESQL.equals(type.name())) { + if (Constants.MYSQL.equals(type.name()) || Constants.POSTGRESQL.equals(type.name()) || Constants.CLICKHOUSE.equals(type.name())) { separator = "&"; } else if (Constants.HIVE.equals(type.name()) || Constants.SPARK.equals(type.name())) { separator = ";"; @@ -479,6 +486,9 @@ public class DataSourceService extends BaseService{ } sb.deleteCharAt(sb.length() - 1); } + } else if (Constants.CLICKHOUSE.equals(type.name())) { + sb.append(Constants.JDBC_CLICKHOUSE); + sb.append(host).append(":").append(port); } return sb.toString(); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java index bdeb6d689e..aa25da8f11 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java @@ -82,6 +82,7 @@ public class Constants { public static final String ORG_POSTGRESQL_DRIVER = "org.postgresql.Driver"; public static final String COM_MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver"; public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"; + public static final String COM_CLICKHOUSE_JDBC_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; /** * database type @@ -90,6 +91,7 @@ public class Constants { public static final String POSTGRESQL = "POSTGRESQL"; public static final String HIVE = "HIVE"; public static final String SPARK = "SPARK"; + public static final String CLICKHOUSE = "CLICKHOUSE"; /** * jdbc url @@ -97,6 +99,7 @@ public class Constants { public static final String JDBC_MYSQL = "jdbc:mysql://"; public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; public static final String JDBC_HIVE_2 = "jdbc:hive2://"; + public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://"; public static final String ADDRESS = "address"; diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index e06b344c4f..35c83bc2b0 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -371,6 +371,21 @@ com.github.oshi oshi-core + + + ru.yandex.clickhouse + clickhouse-jdbc + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index e0e0c399e9..e61bef1018 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -602,15 +602,19 @@ public final class Constants { public static final String JDBC_POSTGRESQL_CLASS_NAME = "org.postgresql.Driver"; /** - * postgresql + * hive */ public static final String JDBC_HIVE_CLASS_NAME = "org.apache.hive.jdbc.HiveDriver"; /** - * postgresql + * spark */ public static final String JDBC_SPARK_CLASS_NAME = "org.apache.hive.jdbc.HiveDriver"; + /** + * ClickHouse + */ + public static final String JDBC_CLICKHOUSE_CLASS_NAME = "ru.yandex.clickhouse.ClickHouseDriver"; /** * spark params constant diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/DbType.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/DbType.java index 70f767444f..bcd7e71dbd 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/enums/DbType.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/DbType.java @@ -25,6 +25,7 @@ public enum DbType { * 1 postgresql * 2 hive * 3 spark + * 4 clickhouse */ - MYSQL, POSTGRESQL, HIVE, SPARK + MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE } diff --git a/escheduler-common/src/main/java/cn/escheduler/common/job/db/ClickHouseDataSource.java b/escheduler-common/src/main/java/cn/escheduler/common/job/db/ClickHouseDataSource.java new file mode 100644 index 0000000000..b4df4d8f5a --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/job/db/ClickHouseDataSource.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.common.job.db; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +/** + * data source of ClickHouse + */ +public class ClickHouseDataSource extends BaseDataSource { + private static final Logger logger = LoggerFactory.getLogger(ClickHouseDataSource.class); + + /** + * gets the JDBC url for the data source connection + * @return + */ + @Override + public String getJdbcUrl() { + String jdbcUrl = getAddress(); + if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { + jdbcUrl += "/"; + } + + jdbcUrl += getDatabase(); + + if (StringUtils.isNotEmpty(getOther())) { + jdbcUrl += "?" + getOther(); + } + + return jdbcUrl; + } + + /** + * test whether the data source can be connected successfully + * @throws Exception + */ + @Override + public void isConnectable() throws Exception { + Connection con = null; + try { + Class.forName("ru.yandex.clickhouse.ClickHouseDriver"); + con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); + } finally { + if (con != null) { + try { + con.close(); + } catch (SQLException e) { + logger.error("ClickHouse datasource try conn close conn error", e); + throw e; + } + } + } + + } +} diff --git a/escheduler-common/src/main/java/cn/escheduler/common/job/db/DataSourceFactory.java b/escheduler-common/src/main/java/cn/escheduler/common/job/db/DataSourceFactory.java index 06858fade3..c694b9c708 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/job/db/DataSourceFactory.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/job/db/DataSourceFactory.java @@ -39,6 +39,8 @@ public class DataSourceFactory { return JSONUtils.parseObject(parameter, HiveDataSource.class); case SPARK: return JSONUtils.parseObject(parameter, SparkDataSource.class); + case CLICKHOUSE: + return JSONUtils.parseObject(parameter, ClickHouseDataSource.class); default: return null; } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/processdure/ProcedureTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/processdure/ProcedureTask.java index 2efe8cbf54..4c3f3f63ad 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/processdure/ProcedureTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/processdure/ProcedureTask.java @@ -22,6 +22,7 @@ import cn.escheduler.common.enums.DbType; import cn.escheduler.common.enums.Direct; import cn.escheduler.common.enums.TaskTimeoutStrategy; import cn.escheduler.common.job.db.BaseDataSource; +import cn.escheduler.common.job.db.ClickHouseDataSource; import cn.escheduler.common.job.db.MySQLDataSource; import cn.escheduler.common.job.db.PostgreDataSource; import cn.escheduler.common.process.Property; @@ -111,6 +112,11 @@ public class ProcedureTask extends AbstractTask { }else if (DbType.POSTGRESQL.name().equals(dataSource.getType().name())){ baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),PostgreDataSource.class); Class.forName(Constants.JDBC_POSTGRESQL_CLASS_NAME); + }else if (DbType.CLICKHOUSE.name().equals(dataSource.getType().name())){ + // NOTE: currently, ClickHouse don't support procedure or UDF yet, + // but still load JDBC driver to keep source code sync with other DB + baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),ClickHouseDataSource.class); + Class.forName(Constants.JDBC_CLICKHOUSE_CLASS_NAME); } // get jdbc connection diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index 36d92d71b5..858e7b8bfc 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -120,6 +120,9 @@ public class SqlTask extends AbstractTask { }else if (DbType.SPARK.name().equals(dataSource.getType().name())){ baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),SparkDataSource.class); Class.forName(Constants.JDBC_SPARK_CLASS_NAME); + }else if (DbType.CLICKHOUSE.name().equals(dataSource.getType().name())){ + baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),ClickHouseDataSource.class); + Class.forName(Constants.JDBC_CLICKHOUSE_CLASS_NAME); } Map sqlParamMap = new HashMap(); diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue index 44a2d8dd40..84a2d05634 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue @@ -6,7 +6,7 @@ diff --git a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue index c06e9dbca8..ceaf118285 100644 --- a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue +++ b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue @@ -13,6 +13,7 @@ POSTGRESQL HVIE SPARK + CLICKHOUSE diff --git a/escheduler-ui/src/js/conf/home/store/dag/state.js b/escheduler-ui/src/js/conf/home/store/dag/state.js index 63b06abef6..e51c2186ec 100644 --- a/escheduler-ui/src/js/conf/home/store/dag/state.js +++ b/escheduler-ui/src/js/conf/home/store/dag/state.js @@ -66,6 +66,11 @@ export default { id: 3, code: 'SPARK', disabled: false + }, + { + id: 4, + code: 'CLICKHOUSE', + disabled: false } ], // Alarm interface diff --git a/escheduler-ui/src/js/conf/home/store/datasource/actions.js b/escheduler-ui/src/js/conf/home/store/datasource/actions.js index 0aa17266f8..4e7bd13d76 100644 --- a/escheduler-ui/src/js/conf/home/store/datasource/actions.js +++ b/escheduler-ui/src/js/conf/home/store/datasource/actions.js @@ -20,7 +20,7 @@ import io from '@/module/io' export default { /** * Data source creation - * @param "type": string,//MYSQL, POSTGRESQL, HIVE + * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE * @param "name": string, * @param "desc": string, * @param "parameter":string //{"address":"jdbc:hive2://192.168.220.189:10000","autoReconnect":"true","characterEncoding":"utf8","database":"default","initialTimeout":3000,"jdbcUrl":"jdbc:hive2://192.168.220.189:10000/default","maxReconnect":10,"password":"","useUnicode":true,"user":"hive"} @@ -49,7 +49,7 @@ export default { }, /** * Query data source list - no paging - * @param "type": string//MYSQL, POSTGRESQL, HIVE + * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE */ getDatasourcesList ({ state }, payload) { return new Promise((resolve, reject) => { diff --git a/pom.xml b/pom.xml index e6f845d0ed..2b0b585106 100644 --- a/pom.xml +++ b/pom.xml @@ -366,6 +366,12 @@ 3.5.0 + + ru.yandex.clickhouse + clickhouse-jdbc + 0.1.52 + +