|
|
|
|
@ -105,16 +105,14 @@ public class SqlTask extends AbstractTask {
|
|
|
|
|
sqlParameters.getUdfs(), |
|
|
|
|
sqlParameters.getShowType(), |
|
|
|
|
sqlParameters.getConnParams()); |
|
|
|
|
|
|
|
|
|
Connection con = null; |
|
|
|
|
List<String> createFuncs = null; |
|
|
|
|
try { |
|
|
|
|
SQLTaskExecutionContext sqlTaskExecutionContext = taskExecutionContext.getSqlTaskExecutionContext(); |
|
|
|
|
// load class
|
|
|
|
|
DataSourceFactory.loadClass(DbType.valueOf(sqlParameters.getType())); |
|
|
|
|
|
|
|
|
|
// get datasource
|
|
|
|
|
baseDataSource = DataSourceFactory.getDatasource(DbType.valueOf(sqlParameters.getType()), |
|
|
|
|
sqlParameters.getConnParams()); |
|
|
|
|
sqlTaskExecutionContext.getConnectionParams()); |
|
|
|
|
|
|
|
|
|
// ready to execute SQL and parameter entity Map
|
|
|
|
|
SqlBinds mainSqlBinds = getSqlAndSqlParamsMap(sqlParameters.getSql()); |
|
|
|
|
@ -129,32 +127,18 @@ public class SqlTask extends AbstractTask {
|
|
|
|
|
.map(this::getSqlAndSqlParamsMap) |
|
|
|
|
.collect(Collectors.toList()); |
|
|
|
|
|
|
|
|
|
// determine if it is UDF
|
|
|
|
|
boolean udfTypeFlag = EnumUtils.isValidEnum(UdfType.class, sqlParameters.getType()) |
|
|
|
|
&& StringUtils.isNotEmpty(sqlParameters.getUdfs()); |
|
|
|
|
if(udfTypeFlag){ |
|
|
|
|
String[] ids = sqlParameters.getUdfs().split(","); |
|
|
|
|
int[] idsArray = new int[ids.length]; |
|
|
|
|
for(int i=0;i<ids.length;i++){ |
|
|
|
|
idsArray[i]=Integer.parseInt(ids[i]); |
|
|
|
|
} |
|
|
|
|
SQLTaskExecutionContext sqlTaskExecutionContext = taskExecutionContext.getSqlTaskExecutionContext(); |
|
|
|
|
createFuncs = UDFUtils.createFuncs(sqlTaskExecutionContext.getUdfFuncList(), taskExecutionContext.getTenantCode(), logger); |
|
|
|
|
} |
|
|
|
|
List<String> createFuncs = UDFUtils.createFuncs(sqlTaskExecutionContext.getUdfFuncList(), |
|
|
|
|
taskExecutionContext.getTenantCode(), |
|
|
|
|
logger); |
|
|
|
|
|
|
|
|
|
// execute sql task
|
|
|
|
|
con = executeFuncAndSql(mainSqlBinds, preStatementSqlBinds, postStatementSqlBinds, createFuncs); |
|
|
|
|
executeFuncAndSql(mainSqlBinds, preStatementSqlBinds, postStatementSqlBinds, createFuncs); |
|
|
|
|
|
|
|
|
|
setExitStatusCode(Constants.EXIT_CODE_SUCCESS); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
logger.error(e.getMessage(), e); |
|
|
|
|
setExitStatusCode(Constants.EXIT_CODE_FAILURE); |
|
|
|
|
logger.error("sql task error", e); |
|
|
|
|
throw e; |
|
|
|
|
} finally { |
|
|
|
|
if (con != null) { |
|
|
|
|
try { |
|
|
|
|
con.close(); |
|
|
|
|
} catch (SQLException e) { |
|
|
|
|
logger.error(e.getMessage(),e); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@ -193,11 +177,11 @@ public class SqlTask extends AbstractTask {
|
|
|
|
|
setSqlParamsMap(sql, rgex, sqlParamsMap, paramsMap); |
|
|
|
|
|
|
|
|
|
// replace the ${} of the SQL statement with the Placeholder
|
|
|
|
|
String formatSql = sql.replaceAll(rgex,"?"); |
|
|
|
|
String formatSql = sql.replaceAll(rgex, "?"); |
|
|
|
|
sqlBuilder.append(formatSql); |
|
|
|
|
|
|
|
|
|
// print repalce sql
|
|
|
|
|
printReplacedSql(sql,formatSql,rgex,sqlParamsMap); |
|
|
|
|
printReplacedSql(sql, formatSql, rgex, sqlParamsMap); |
|
|
|
|
return new SqlBinds(sqlBuilder.toString(), sqlParamsMap); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@ -214,55 +198,59 @@ public class SqlTask extends AbstractTask {
|
|
|
|
|
* @param createFuncs create functions |
|
|
|
|
* @return Connection |
|
|
|
|
*/ |
|
|
|
|
public Connection executeFuncAndSql(SqlBinds mainSqlBinds, |
|
|
|
|
public void executeFuncAndSql(SqlBinds mainSqlBinds, |
|
|
|
|
List<SqlBinds> preStatementsBinds, |
|
|
|
|
List<SqlBinds> postStatementsBinds, |
|
|
|
|
List<String> createFuncs){ |
|
|
|
|
Connection connection = null; |
|
|
|
|
PreparedStatement stmt = null; |
|
|
|
|
ResultSet resultSet = null; |
|
|
|
|
try { |
|
|
|
|
// if upload resource is HDFS and kerberos startup
|
|
|
|
|
CommonUtils.loadKerberosConf(); |
|
|
|
|
|
|
|
|
|
// if hive , load connection params if exists
|
|
|
|
|
if (HIVE == DbType.valueOf(sqlParameters.getType())) { |
|
|
|
|
Properties paramProp = new Properties(); |
|
|
|
|
paramProp.setProperty(USER, baseDataSource.getUser()); |
|
|
|
|
paramProp.setProperty(PASSWORD, baseDataSource.getPassword()); |
|
|
|
|
Map<String, String> connParamMap = CollectionUtils.stringToMap(sqlParameters.getConnParams(), |
|
|
|
|
SEMICOLON, |
|
|
|
|
HIVE_CONF); |
|
|
|
|
paramProp.putAll(connParamMap); |
|
|
|
|
|
|
|
|
|
connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), |
|
|
|
|
paramProp); |
|
|
|
|
}else{ |
|
|
|
|
connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), |
|
|
|
|
baseDataSource.getUser(), |
|
|
|
|
baseDataSource.getPassword()); |
|
|
|
|
} |
|
|
|
|
// create connection
|
|
|
|
|
connection = createConnection(); |
|
|
|
|
|
|
|
|
|
// create temp function
|
|
|
|
|
if (CollectionUtils.isNotEmpty(createFuncs)) { |
|
|
|
|
try (Statement funcStmt = connection.createStatement()) { |
|
|
|
|
for (String createFunc : createFuncs) { |
|
|
|
|
logger.info("hive create function sql: {}", createFunc); |
|
|
|
|
funcStmt.execute(createFunc); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
createTempFunction(connection,createFuncs); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
for (SqlBinds sqlBind: preStatementsBinds) { |
|
|
|
|
try (PreparedStatement stmt = prepareStatementAndBind(connection, sqlBind)) { |
|
|
|
|
int result = stmt.executeUpdate(); |
|
|
|
|
logger.info("pre statement execute result: {}, for sql: {}",result,sqlBind.getSql()); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
// pre sql
|
|
|
|
|
preSql(connection,preStatementsBinds); |
|
|
|
|
|
|
|
|
|
try (PreparedStatement stmt = prepareStatementAndBind(connection, mainSqlBinds); |
|
|
|
|
ResultSet resultSet = stmt.executeQuery()) { |
|
|
|
|
|
|
|
|
|
stmt = prepareStatementAndBind(connection, mainSqlBinds); |
|
|
|
|
resultSet = stmt.executeQuery(); |
|
|
|
|
// decide whether to executeQuery or executeUpdate based on sqlType
|
|
|
|
|
if (sqlParameters.getSqlType() == SqlType.QUERY.ordinal()) { |
|
|
|
|
// query statements need to be convert to JsonArray and inserted into Alert to send
|
|
|
|
|
resultProcess(resultSet); |
|
|
|
|
|
|
|
|
|
} else if (sqlParameters.getSqlType() == SqlType.NON_QUERY.ordinal()) { |
|
|
|
|
// non query statement
|
|
|
|
|
stmt.executeUpdate(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
postSql(connection,postStatementsBinds); |
|
|
|
|
|
|
|
|
|
} catch (Exception e) { |
|
|
|
|
logger.error("execute sql error",e); |
|
|
|
|
throw new RuntimeException("execute sql error"); |
|
|
|
|
} finally { |
|
|
|
|
close(resultSet,stmt,connection); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* result process |
|
|
|
|
* |
|
|
|
|
* @param resultSet resultSet |
|
|
|
|
* @throws Exception |
|
|
|
|
*/ |
|
|
|
|
private void resultProcess(ResultSet resultSet) throws Exception{ |
|
|
|
|
JSONArray resultJSONArray = new JSONArray(); |
|
|
|
|
ResultSetMetaData md = resultSet.getMetaData(); |
|
|
|
|
int num = md.getColumnCount(); |
|
|
|
|
@ -277,7 +265,7 @@ public class SqlTask extends AbstractTask {
|
|
|
|
|
logger.debug("execute sql : {}", JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); |
|
|
|
|
|
|
|
|
|
// if there is a result set
|
|
|
|
|
if ( !resultJSONArray.isEmpty() ) { |
|
|
|
|
if (!resultJSONArray.isEmpty() ) { |
|
|
|
|
if (StringUtils.isNotEmpty(sqlParameters.getTitle())) { |
|
|
|
|
sendAttachment(sqlParameters.getTitle(), |
|
|
|
|
JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); |
|
|
|
|
@ -286,33 +274,119 @@ public class SqlTask extends AbstractTask {
|
|
|
|
|
JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
exitStatusCode = 0; |
|
|
|
|
/** |
|
|
|
|
* pre sql |
|
|
|
|
* |
|
|
|
|
* @param connection connection |
|
|
|
|
* @param preStatementsBinds preStatementsBinds |
|
|
|
|
*/ |
|
|
|
|
private void preSql(Connection connection, |
|
|
|
|
List<SqlBinds> preStatementsBinds) throws Exception{ |
|
|
|
|
for (SqlBinds sqlBind: preStatementsBinds) { |
|
|
|
|
try (PreparedStatement pstmt = prepareStatementAndBind(connection, sqlBind)){ |
|
|
|
|
int result = pstmt.executeUpdate(); |
|
|
|
|
logger.info("pre statement execute result: {}, for sql: {}",result,sqlBind.getSql()); |
|
|
|
|
|
|
|
|
|
} else if (sqlParameters.getSqlType() == SqlType.NON_QUERY.ordinal()) { |
|
|
|
|
// non query statement
|
|
|
|
|
stmt.executeUpdate(); |
|
|
|
|
exitStatusCode = 0; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* post psql |
|
|
|
|
* |
|
|
|
|
* @param connection connection |
|
|
|
|
* @param postStatementsBinds postStatementsBinds |
|
|
|
|
* @throws Exception |
|
|
|
|
*/ |
|
|
|
|
private void postSql(Connection connection, |
|
|
|
|
List<SqlBinds> postStatementsBinds) throws Exception{ |
|
|
|
|
for (SqlBinds sqlBind: postStatementsBinds) { |
|
|
|
|
try (PreparedStatement stmt = prepareStatementAndBind(connection, sqlBind)) { |
|
|
|
|
int result = stmt.executeUpdate(); |
|
|
|
|
try (PreparedStatement pstmt = prepareStatementAndBind(connection, sqlBind)){ |
|
|
|
|
int result = pstmt.executeUpdate(); |
|
|
|
|
logger.info("post statement execute result: {},for sql: {}",result,sqlBind.getSql()); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
logger.error(e.getMessage(),e); |
|
|
|
|
throw new RuntimeException(e.getMessage()); |
|
|
|
|
} finally { |
|
|
|
|
} |
|
|
|
|
/** |
|
|
|
|
* create temp function |
|
|
|
|
* |
|
|
|
|
* @param connection connection |
|
|
|
|
* @param createFuncs createFuncs |
|
|
|
|
* @throws Exception |
|
|
|
|
*/ |
|
|
|
|
private void createTempFunction(Connection connection, |
|
|
|
|
List<String> createFuncs) throws Exception{ |
|
|
|
|
try (Statement funcStmt = connection.createStatement()) { |
|
|
|
|
for (String createFunc : createFuncs) { |
|
|
|
|
logger.info("hive create function sql: {}", createFunc); |
|
|
|
|
funcStmt.execute(createFunc); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
/** |
|
|
|
|
* create connection |
|
|
|
|
* |
|
|
|
|
* @return connection |
|
|
|
|
* @throws Exception |
|
|
|
|
*/ |
|
|
|
|
private Connection createConnection() throws Exception{ |
|
|
|
|
// if hive , load connection params if exists
|
|
|
|
|
Connection connection = null; |
|
|
|
|
if (HIVE == DbType.valueOf(sqlParameters.getType())) { |
|
|
|
|
Properties paramProp = new Properties(); |
|
|
|
|
paramProp.setProperty(USER, baseDataSource.getUser()); |
|
|
|
|
paramProp.setProperty(PASSWORD, baseDataSource.getPassword()); |
|
|
|
|
Map<String, String> connParamMap = CollectionUtils.stringToMap(sqlParameters.getConnParams(), |
|
|
|
|
SEMICOLON, |
|
|
|
|
HIVE_CONF); |
|
|
|
|
paramProp.putAll(connParamMap); |
|
|
|
|
|
|
|
|
|
connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), |
|
|
|
|
paramProp); |
|
|
|
|
}else{ |
|
|
|
|
connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), |
|
|
|
|
baseDataSource.getUser(), |
|
|
|
|
baseDataSource.getPassword()); |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
return connection; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* close jdbc resource |
|
|
|
|
* |
|
|
|
|
* @param resultSet resultSet |
|
|
|
|
* @param pstmt pstmt |
|
|
|
|
* @param connection connection |
|
|
|
|
*/ |
|
|
|
|
private void close(ResultSet resultSet, |
|
|
|
|
PreparedStatement pstmt, |
|
|
|
|
Connection connection){ |
|
|
|
|
if (resultSet != null){ |
|
|
|
|
try { |
|
|
|
|
connection.close(); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
logger.error(e.getMessage(), e); |
|
|
|
|
} catch (SQLException e) { |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (pstmt != null){ |
|
|
|
|
try { |
|
|
|
|
connection.close(); |
|
|
|
|
} catch (SQLException e) { |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (connection != null){ |
|
|
|
|
try { |
|
|
|
|
connection.close(); |
|
|
|
|
} catch (SQLException e) { |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
return connection; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
@ -326,7 +400,7 @@ public class SqlTask extends AbstractTask {
|
|
|
|
|
// is the timeout set
|
|
|
|
|
boolean timeoutFlag = TaskTimeoutStrategy.of(taskExecutionContext.getTaskTimeoutStrategy()) == TaskTimeoutStrategy.FAILED || |
|
|
|
|
TaskTimeoutStrategy.of(taskExecutionContext.getTaskTimeoutStrategy()) == TaskTimeoutStrategy.WARNFAILED; |
|
|
|
|
try (PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql())) { |
|
|
|
|
PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql()); |
|
|
|
|
if(timeoutFlag){ |
|
|
|
|
stmt.setQueryTimeout(taskExecutionContext.getTaskTimeout()); |
|
|
|
|
} |
|
|
|
|
@ -340,7 +414,6 @@ public class SqlTask extends AbstractTask {
|
|
|
|
|
logger.info("prepare statement replace sql : {} ", stmt); |
|
|
|
|
return stmt; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* send mail as an attachment |
|
|
|
|
|
qiaozhanwei
5 years ago
committed by
GitHub