From 2d3b7baa1a256bc6ce75d94de39abb012133b88c Mon Sep 17 00:00:00 2001 From: Baoqi Date: Sat, 13 Apr 2019 15:42:37 +0800 Subject: [PATCH] support Oracle Database --- .../api/service/DataSourceService.java | 17 +++-- .../cn/escheduler/api/utils/Constants.java | 3 + .../java/cn/escheduler/common/Constants.java | 5 ++ .../cn/escheduler/common/enums/DbType.java | 3 +- .../common/job/db/DataSourceFactory.java | 2 + .../common/job/db/OracleDataSource.java | 75 +++++++++++++++++++ .../task/processdure/ProcedureTask.java | 4 + .../server/worker/task/sql/SqlTask.java | 3 + .../server/worker/sql/SqlExecutorTest.java | 15 +++- .../dag/_source/formModel/tasks/procedure.vue | 2 +- .../pages/list/_source/createDataSource.vue | 1 + .../src/js/conf/home/store/dag/state.js | 5 ++ .../js/conf/home/store/datasource/actions.js | 4 +- 13 files changed, 127 insertions(+), 12 deletions(-) create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/job/db/OracleDataSource.java diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 58c24bddc1..a717dd1342 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -213,12 +213,9 @@ public class DataSourceService extends BaseService{ separator = ";"; break; case MYSQL: - separator = "&"; - break; case POSTGRESQL: - separator = "&"; - break; case CLICKHOUSE: + case ORACLE: separator = "&"; break; default: @@ -375,6 +372,10 @@ public class DataSourceService extends BaseService{ datasource = JSONObject.parseObject(parameter, ClickHouseDataSource.class); Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); break; + case ORACLE: + datasource = JSONObject.parseObject(parameter, OracleDataSource.class); + Class.forName(Constants.COM_ORACLE_JDBC_DRIVER); + break; default: break; } @@ -441,7 +442,10 @@ public class DataSourceService extends BaseService{ String address = buildAddress(type, host, port); String jdbcUrl = address + "/" + database; String separator = ""; - if (Constants.MYSQL.equals(type.name()) || Constants.POSTGRESQL.equals(type.name()) || Constants.CLICKHOUSE.equals(type.name())) { + if (Constants.MYSQL.equals(type.name()) + || Constants.POSTGRESQL.equals(type.name()) + || Constants.CLICKHOUSE.equals(type.name()) + || Constants.ORACLE.equals(type.name())) { separator = "&"; } else if (Constants.HIVE.equals(type.name()) || Constants.SPARK.equals(type.name())) { separator = ";"; @@ -495,6 +499,9 @@ public class DataSourceService extends BaseService{ } else if (Constants.CLICKHOUSE.equals(type.name())) { sb.append(Constants.JDBC_CLICKHOUSE); sb.append(host).append(":").append(port); + } else if (Constants.ORACLE.equals(type.name())) { + sb.append(Constants.JDBC_ORACLE); + sb.append(host).append(":").append(port); } return sb.toString(); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java index aa25da8f11..0335026bb6 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java @@ -83,6 +83,7 @@ public class Constants { public static final String COM_MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver"; public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"; public static final String COM_CLICKHOUSE_JDBC_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; + public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.driver.OracleDriver"; /** * database type @@ -92,6 +93,7 @@ public class Constants { public static final String HIVE = "HIVE"; public static final String SPARK = "SPARK"; public static final String CLICKHOUSE = "CLICKHOUSE"; + public static final String ORACLE = "ORACLE"; /** * jdbc url @@ -100,6 +102,7 @@ public class Constants { public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; public static final String JDBC_HIVE_2 = "jdbc:hive2://"; public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://"; + public static final String JDBC_ORACLE = "jdbc:oracle:thin:@//"; public static final String ADDRESS = "address"; diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index e61bef1018..c0cd2290f6 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -616,6 +616,11 @@ public final class Constants { */ public static final String JDBC_CLICKHOUSE_CLASS_NAME = "ru.yandex.clickhouse.ClickHouseDriver"; + /** + * Oracle + */ + public static final String JDBC_ORACLE_CLASS_NAME = "oracle.jdbc.driver.OracleDriver"; + /** * spark params constant */ diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/DbType.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/DbType.java index bcd7e71dbd..df8e86a182 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/enums/DbType.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/DbType.java @@ -26,6 +26,7 @@ public enum DbType { * 2 hive * 3 spark * 4 clickhouse + * 5 oracle */ - MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE + MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE } diff --git a/escheduler-common/src/main/java/cn/escheduler/common/job/db/DataSourceFactory.java b/escheduler-common/src/main/java/cn/escheduler/common/job/db/DataSourceFactory.java index c694b9c708..4008645074 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/job/db/DataSourceFactory.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/job/db/DataSourceFactory.java @@ -41,6 +41,8 @@ public class DataSourceFactory { return JSONUtils.parseObject(parameter, SparkDataSource.class); case CLICKHOUSE: return JSONUtils.parseObject(parameter, ClickHouseDataSource.class); + case ORACLE: + return JSONUtils.parseObject(parameter, OracleDataSource.class); default: return null; } diff --git a/escheduler-common/src/main/java/cn/escheduler/common/job/db/OracleDataSource.java b/escheduler-common/src/main/java/cn/escheduler/common/job/db/OracleDataSource.java new file mode 100644 index 0000000000..5e245a590c --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/job/db/OracleDataSource.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.common.job.db; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +/** + * data source of Oracle + */ +public class OracleDataSource extends BaseDataSource { + private static final Logger logger = LoggerFactory.getLogger(OracleDataSource.class); + + /** + * gets the JDBC url for the data source connection + * @return + */ + @Override + public String getJdbcUrl() { + String jdbcUrl = getAddress(); + if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { + jdbcUrl += "/"; + } + + jdbcUrl += getDatabase(); + + if (StringUtils.isNotEmpty(getOther())) { + jdbcUrl += "?" + getOther(); + } + + return jdbcUrl; + } + + /** + * test whether the data source can be connected successfully + * @throws Exception + */ + @Override + public void isConnectable() throws Exception { + Connection con = null; + try { + Class.forName("oracle.jdbc.driver.OracleDriver"); + con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); + } finally { + if (con != null) { + try { + con.close(); + } catch (SQLException e) { + logger.error("Oracle datasource try conn close conn error", e); + throw e; + } + } + } + + } +} diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/processdure/ProcedureTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/processdure/ProcedureTask.java index 4c3f3f63ad..ee592b89e7 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/processdure/ProcedureTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/processdure/ProcedureTask.java @@ -24,6 +24,7 @@ import cn.escheduler.common.enums.TaskTimeoutStrategy; import cn.escheduler.common.job.db.BaseDataSource; import cn.escheduler.common.job.db.ClickHouseDataSource; import cn.escheduler.common.job.db.MySQLDataSource; +import cn.escheduler.common.job.db.OracleDataSource; import cn.escheduler.common.job.db.PostgreDataSource; import cn.escheduler.common.process.Property; import cn.escheduler.common.task.AbstractParameters; @@ -117,6 +118,9 @@ public class ProcedureTask extends AbstractTask { // but still load JDBC driver to keep source code sync with other DB baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),ClickHouseDataSource.class); Class.forName(Constants.JDBC_CLICKHOUSE_CLASS_NAME); + }else if (DbType.ORACLE.name().equals(dataSource.getType().name())){ + baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(), OracleDataSource.class); + Class.forName(Constants.JDBC_ORACLE_CLASS_NAME); } // get jdbc connection diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index 858e7b8bfc..2c30f328c0 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -123,6 +123,9 @@ public class SqlTask extends AbstractTask { }else if (DbType.CLICKHOUSE.name().equals(dataSource.getType().name())){ baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),ClickHouseDataSource.class); Class.forName(Constants.JDBC_CLICKHOUSE_CLASS_NAME); + }else if (DbType.ORACLE.name().equals(dataSource.getType().name())){ + baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),OracleDataSource.class); + Class.forName(Constants.JDBC_ORACLE_CLASS_NAME); } Map sqlParamMap = new HashMap(); diff --git a/escheduler-server/src/test/java/cn/escheduler/server/worker/sql/SqlExecutorTest.java b/escheduler-server/src/test/java/cn/escheduler/server/worker/sql/SqlExecutorTest.java index 5565cd6a08..472dea73bc 100644 --- a/escheduler-server/src/test/java/cn/escheduler/server/worker/sql/SqlExecutorTest.java +++ b/escheduler-server/src/test/java/cn/escheduler/server/worker/sql/SqlExecutorTest.java @@ -55,7 +55,7 @@ public class SqlExecutorTest { String nodeName = "mysql sql test"; String taskAppId = "51_11282_263978"; String tenantCode = "hdfs"; - Integer taskInstId = 263978; + int taskInstId = 263978; sharedTestSqlTask(nodeName, taskAppId, tenantCode, taskInstId); } @@ -64,7 +64,16 @@ public class SqlExecutorTest { String nodeName = "ClickHouse sql test"; String taskAppId = "1_11_20"; String tenantCode = "default"; - Integer taskInstId = 20; + int taskInstId = 20; + sharedTestSqlTask(nodeName, taskAppId, tenantCode, taskInstId); + } + + @Test + public void testOracle() throws Exception { + String nodeName = "oracle sql test"; + String taskAppId = "2_13_25"; + String tenantCode = "demo"; + int taskInstId = 25; sharedTestSqlTask(nodeName, taskAppId, tenantCode, taskInstId); } @@ -76,7 +85,7 @@ public class SqlExecutorTest { * @param taskInstId task instance id * @throws Exception */ - private void sharedTestSqlTask(String nodeName, String taskAppId, String tenantCode, Integer taskInstId) throws Exception { + private void sharedTestSqlTask(String nodeName, String taskAppId, String tenantCode, int taskInstId) throws Exception { TaskProps taskProps = new TaskProps(); taskProps.setTaskDir(""); // processDefineId_processInstanceId_taskInstanceId diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue index 84a2d05634..8ffe5b1139 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue @@ -6,7 +6,7 @@ diff --git a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue index 96e453c318..4a8de42802 100644 --- a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue +++ b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue @@ -14,6 +14,7 @@ HIVE SPARK CLICKHOUSE + ORACLE diff --git a/escheduler-ui/src/js/conf/home/store/dag/state.js b/escheduler-ui/src/js/conf/home/store/dag/state.js index e51c2186ec..1cae24a84e 100644 --- a/escheduler-ui/src/js/conf/home/store/dag/state.js +++ b/escheduler-ui/src/js/conf/home/store/dag/state.js @@ -71,6 +71,11 @@ export default { id: 4, code: 'CLICKHOUSE', disabled: false + }, + { + id: 5, + code: 'ORACLE', + disabled: false } ], // Alarm interface diff --git a/escheduler-ui/src/js/conf/home/store/datasource/actions.js b/escheduler-ui/src/js/conf/home/store/datasource/actions.js index 4e7bd13d76..b19372666f 100644 --- a/escheduler-ui/src/js/conf/home/store/datasource/actions.js +++ b/escheduler-ui/src/js/conf/home/store/datasource/actions.js @@ -20,7 +20,7 @@ import io from '@/module/io' export default { /** * Data source creation - * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE + * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE * @param "name": string, * @param "desc": string, * @param "parameter":string //{"address":"jdbc:hive2://192.168.220.189:10000","autoReconnect":"true","characterEncoding":"utf8","database":"default","initialTimeout":3000,"jdbcUrl":"jdbc:hive2://192.168.220.189:10000/default","maxReconnect":10,"password":"","useUnicode":true,"user":"hive"} @@ -49,7 +49,7 @@ export default { }, /** * Query data source list - no paging - * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE + * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE */ getDatasourcesList ({ state }, payload) { return new Promise((resolve, reject) => {