Browse Source

support Oracle Database

pull/1/MERGE
Baoqi 6 years ago
parent
commit
2d3b7baa1a
  1. 17
      escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java
  2. 3
      escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java
  3. 5
      escheduler-common/src/main/java/cn/escheduler/common/Constants.java
  4. 3
      escheduler-common/src/main/java/cn/escheduler/common/enums/DbType.java
  5. 2
      escheduler-common/src/main/java/cn/escheduler/common/job/db/DataSourceFactory.java
  6. 75
      escheduler-common/src/main/java/cn/escheduler/common/job/db/OracleDataSource.java
  7. 4
      escheduler-server/src/main/java/cn/escheduler/server/worker/task/processdure/ProcedureTask.java
  8. 3
      escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java
  9. 15
      escheduler-server/src/test/java/cn/escheduler/server/worker/sql/SqlExecutorTest.java
  10. 2
      escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue
  11. 1
      escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue
  12. 5
      escheduler-ui/src/js/conf/home/store/dag/state.js
  13. 4
      escheduler-ui/src/js/conf/home/store/datasource/actions.js

17
escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java

@ -213,12 +213,9 @@ public class DataSourceService extends BaseService{
separator = ";"; separator = ";";
break; break;
case MYSQL: case MYSQL:
separator = "&";
break;
case POSTGRESQL: case POSTGRESQL:
separator = "&";
break;
case CLICKHOUSE: case CLICKHOUSE:
case ORACLE:
separator = "&"; separator = "&";
break; break;
default: default:
@ -375,6 +372,10 @@ public class DataSourceService extends BaseService{
datasource = JSONObject.parseObject(parameter, ClickHouseDataSource.class); datasource = JSONObject.parseObject(parameter, ClickHouseDataSource.class);
Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER);
break; break;
case ORACLE:
datasource = JSONObject.parseObject(parameter, OracleDataSource.class);
Class.forName(Constants.COM_ORACLE_JDBC_DRIVER);
break;
default: default:
break; break;
} }
@ -441,7 +442,10 @@ public class DataSourceService extends BaseService{
String address = buildAddress(type, host, port); String address = buildAddress(type, host, port);
String jdbcUrl = address + "/" + database; String jdbcUrl = address + "/" + database;
String separator = ""; String separator = "";
if (Constants.MYSQL.equals(type.name()) || Constants.POSTGRESQL.equals(type.name()) || Constants.CLICKHOUSE.equals(type.name())) { if (Constants.MYSQL.equals(type.name())
|| Constants.POSTGRESQL.equals(type.name())
|| Constants.CLICKHOUSE.equals(type.name())
|| Constants.ORACLE.equals(type.name())) {
separator = "&"; separator = "&";
} else if (Constants.HIVE.equals(type.name()) || Constants.SPARK.equals(type.name())) { } else if (Constants.HIVE.equals(type.name()) || Constants.SPARK.equals(type.name())) {
separator = ";"; separator = ";";
@ -495,6 +499,9 @@ public class DataSourceService extends BaseService{
} else if (Constants.CLICKHOUSE.equals(type.name())) { } else if (Constants.CLICKHOUSE.equals(type.name())) {
sb.append(Constants.JDBC_CLICKHOUSE); sb.append(Constants.JDBC_CLICKHOUSE);
sb.append(host).append(":").append(port); sb.append(host).append(":").append(port);
} else if (Constants.ORACLE.equals(type.name())) {
sb.append(Constants.JDBC_ORACLE);
sb.append(host).append(":").append(port);
} }
return sb.toString(); return sb.toString();

3
escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java

@ -83,6 +83,7 @@ public class Constants {
public static final String COM_MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver"; public static final String COM_MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver";
public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"; public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver";
public static final String COM_CLICKHOUSE_JDBC_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; public static final String COM_CLICKHOUSE_JDBC_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver";
public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.driver.OracleDriver";
/** /**
* database type * database type
@ -92,6 +93,7 @@ public class Constants {
public static final String HIVE = "HIVE"; public static final String HIVE = "HIVE";
public static final String SPARK = "SPARK"; public static final String SPARK = "SPARK";
public static final String CLICKHOUSE = "CLICKHOUSE"; public static final String CLICKHOUSE = "CLICKHOUSE";
public static final String ORACLE = "ORACLE";
/** /**
* jdbc url * jdbc url
@ -100,6 +102,7 @@ public class Constants {
public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; public static final String JDBC_POSTGRESQL = "jdbc:postgresql://";
public static final String JDBC_HIVE_2 = "jdbc:hive2://"; public static final String JDBC_HIVE_2 = "jdbc:hive2://";
public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://"; public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://";
public static final String JDBC_ORACLE = "jdbc:oracle:thin:@//";
public static final String ADDRESS = "address"; public static final String ADDRESS = "address";

5
escheduler-common/src/main/java/cn/escheduler/common/Constants.java

@ -616,6 +616,11 @@ public final class Constants {
*/ */
public static final String JDBC_CLICKHOUSE_CLASS_NAME = "ru.yandex.clickhouse.ClickHouseDriver"; public static final String JDBC_CLICKHOUSE_CLASS_NAME = "ru.yandex.clickhouse.ClickHouseDriver";
/**
* Oracle
*/
public static final String JDBC_ORACLE_CLASS_NAME = "oracle.jdbc.driver.OracleDriver";
/** /**
* spark params constant * spark params constant
*/ */

3
escheduler-common/src/main/java/cn/escheduler/common/enums/DbType.java

@ -26,6 +26,7 @@ public enum DbType {
* 2 hive * 2 hive
* 3 spark * 3 spark
* 4 clickhouse * 4 clickhouse
* 5 oracle
*/ */
MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE
} }

2
escheduler-common/src/main/java/cn/escheduler/common/job/db/DataSourceFactory.java

@ -41,6 +41,8 @@ public class DataSourceFactory {
return JSONUtils.parseObject(parameter, SparkDataSource.class); return JSONUtils.parseObject(parameter, SparkDataSource.class);
case CLICKHOUSE: case CLICKHOUSE:
return JSONUtils.parseObject(parameter, ClickHouseDataSource.class); return JSONUtils.parseObject(parameter, ClickHouseDataSource.class);
case ORACLE:
return JSONUtils.parseObject(parameter, OracleDataSource.class);
default: default:
return null; return null;
} }

75
escheduler-common/src/main/java/cn/escheduler/common/job/db/OracleDataSource.java

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.escheduler.common.job.db;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
/**
* data source of Oracle
*/
public class OracleDataSource extends BaseDataSource {
private static final Logger logger = LoggerFactory.getLogger(OracleDataSource.class);
/**
* gets the JDBC url for the data source connection
* @return
*/
@Override
public String getJdbcUrl() {
String jdbcUrl = getAddress();
if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) {
jdbcUrl += "/";
}
jdbcUrl += getDatabase();
if (StringUtils.isNotEmpty(getOther())) {
jdbcUrl += "?" + getOther();
}
return jdbcUrl;
}
/**
* test whether the data source can be connected successfully
* @throws Exception
*/
@Override
public void isConnectable() throws Exception {
Connection con = null;
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
} finally {
if (con != null) {
try {
con.close();
} catch (SQLException e) {
logger.error("Oracle datasource try conn close conn error", e);
throw e;
}
}
}
}
}

4
escheduler-server/src/main/java/cn/escheduler/server/worker/task/processdure/ProcedureTask.java

@ -24,6 +24,7 @@ import cn.escheduler.common.enums.TaskTimeoutStrategy;
import cn.escheduler.common.job.db.BaseDataSource; import cn.escheduler.common.job.db.BaseDataSource;
import cn.escheduler.common.job.db.ClickHouseDataSource; import cn.escheduler.common.job.db.ClickHouseDataSource;
import cn.escheduler.common.job.db.MySQLDataSource; import cn.escheduler.common.job.db.MySQLDataSource;
import cn.escheduler.common.job.db.OracleDataSource;
import cn.escheduler.common.job.db.PostgreDataSource; import cn.escheduler.common.job.db.PostgreDataSource;
import cn.escheduler.common.process.Property; import cn.escheduler.common.process.Property;
import cn.escheduler.common.task.AbstractParameters; import cn.escheduler.common.task.AbstractParameters;
@ -117,6 +118,9 @@ public class ProcedureTask extends AbstractTask {
// but still load JDBC driver to keep source code sync with other DB // but still load JDBC driver to keep source code sync with other DB
baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),ClickHouseDataSource.class); baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),ClickHouseDataSource.class);
Class.forName(Constants.JDBC_CLICKHOUSE_CLASS_NAME); Class.forName(Constants.JDBC_CLICKHOUSE_CLASS_NAME);
}else if (DbType.ORACLE.name().equals(dataSource.getType().name())){
baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(), OracleDataSource.class);
Class.forName(Constants.JDBC_ORACLE_CLASS_NAME);
} }
// get jdbc connection // get jdbc connection

3
escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java

@ -123,6 +123,9 @@ public class SqlTask extends AbstractTask {
}else if (DbType.CLICKHOUSE.name().equals(dataSource.getType().name())){ }else if (DbType.CLICKHOUSE.name().equals(dataSource.getType().name())){
baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),ClickHouseDataSource.class); baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),ClickHouseDataSource.class);
Class.forName(Constants.JDBC_CLICKHOUSE_CLASS_NAME); Class.forName(Constants.JDBC_CLICKHOUSE_CLASS_NAME);
}else if (DbType.ORACLE.name().equals(dataSource.getType().name())){
baseDataSource = JSONObject.parseObject(dataSource.getConnectionParams(),OracleDataSource.class);
Class.forName(Constants.JDBC_ORACLE_CLASS_NAME);
} }
Map<Integer,Property> sqlParamMap = new HashMap<Integer,Property>(); Map<Integer,Property> sqlParamMap = new HashMap<Integer,Property>();

15
escheduler-server/src/test/java/cn/escheduler/server/worker/sql/SqlExecutorTest.java

@ -55,7 +55,7 @@ public class SqlExecutorTest {
String nodeName = "mysql sql test"; String nodeName = "mysql sql test";
String taskAppId = "51_11282_263978"; String taskAppId = "51_11282_263978";
String tenantCode = "hdfs"; String tenantCode = "hdfs";
Integer taskInstId = 263978; int taskInstId = 263978;
sharedTestSqlTask(nodeName, taskAppId, tenantCode, taskInstId); sharedTestSqlTask(nodeName, taskAppId, tenantCode, taskInstId);
} }
@ -64,7 +64,16 @@ public class SqlExecutorTest {
String nodeName = "ClickHouse sql test"; String nodeName = "ClickHouse sql test";
String taskAppId = "1_11_20"; String taskAppId = "1_11_20";
String tenantCode = "default"; String tenantCode = "default";
Integer taskInstId = 20; int taskInstId = 20;
sharedTestSqlTask(nodeName, taskAppId, tenantCode, taskInstId);
}
@Test
public void testOracle() throws Exception {
String nodeName = "oracle sql test";
String taskAppId = "2_13_25";
String tenantCode = "demo";
int taskInstId = 25;
sharedTestSqlTask(nodeName, taskAppId, tenantCode, taskInstId); sharedTestSqlTask(nodeName, taskAppId, tenantCode, taskInstId);
} }
@ -76,7 +85,7 @@ public class SqlExecutorTest {
* @param taskInstId task instance id * @param taskInstId task instance id
* @throws Exception * @throws Exception
*/ */
private void sharedTestSqlTask(String nodeName, String taskAppId, String tenantCode, Integer taskInstId) throws Exception { private void sharedTestSqlTask(String nodeName, String taskAppId, String tenantCode, int taskInstId) throws Exception {
TaskProps taskProps = new TaskProps(); TaskProps taskProps = new TaskProps();
taskProps.setTaskDir(""); taskProps.setTaskDir("");
// processDefineId_processInstanceId_taskInstanceId // processDefineId_processInstanceId_taskInstanceId

2
escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/procedure.vue

@ -6,7 +6,7 @@
<m-datasource <m-datasource
ref="refDs" ref="refDs"
@on-dsData="_onDsData" @on-dsData="_onDsData"
:supportType="['MYSQL','POSTGRESQL','CLICKHOUSE']" :supportType="['MYSQL','POSTGRESQL','CLICKHOUSE', 'ORACLE']"
:data="{ type:type,datasource:datasource }"> :data="{ type:type,datasource:datasource }">
</m-datasource> </m-datasource>
</div> </div>

1
escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue

@ -14,6 +14,7 @@
<x-radio :label="'HIVE'">HIVE</x-radio> <x-radio :label="'HIVE'">HIVE</x-radio>
<x-radio :label="'SPARK'">SPARK</x-radio> <x-radio :label="'SPARK'">SPARK</x-radio>
<x-radio :label="'CLICKHOUSE'">CLICKHOUSE</x-radio> <x-radio :label="'CLICKHOUSE'">CLICKHOUSE</x-radio>
<x-radio :label="'ORACLE'">ORACLE</x-radio>
</x-radio-group> </x-radio-group>
</template> </template>
</m-list-box-f> </m-list-box-f>

5
escheduler-ui/src/js/conf/home/store/dag/state.js

@ -71,6 +71,11 @@ export default {
id: 4, id: 4,
code: 'CLICKHOUSE', code: 'CLICKHOUSE',
disabled: false disabled: false
},
{
id: 5,
code: 'ORACLE',
disabled: false
} }
], ],
// Alarm interface // Alarm interface

4
escheduler-ui/src/js/conf/home/store/datasource/actions.js

@ -20,7 +20,7 @@ import io from '@/module/io'
export default { export default {
/** /**
* Data source creation * Data source creation
* @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE
* @param "name": string, * @param "name": string,
* @param "desc": string, * @param "desc": string,
* @param "parameter":string //{"address":"jdbc:hive2://192.168.220.189:10000","autoReconnect":"true","characterEncoding":"utf8","database":"default","initialTimeout":3000,"jdbcUrl":"jdbc:hive2://192.168.220.189:10000/default","maxReconnect":10,"password":"","useUnicode":true,"user":"hive"} * @param "parameter":string //{"address":"jdbc:hive2://192.168.220.189:10000","autoReconnect":"true","characterEncoding":"utf8","database":"default","initialTimeout":3000,"jdbcUrl":"jdbc:hive2://192.168.220.189:10000/default","maxReconnect":10,"password":"","useUnicode":true,"user":"hive"}
@ -49,7 +49,7 @@ export default {
}, },
/** /**
* Query data source list - no paging * Query data source list - no paging
* @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE
*/ */
getDatasourcesList ({ state }, payload) { getDatasourcesList ({ state }, payload) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {

Loading…
Cancel
Save