From c08b5189fa428dbeee8d3416f28b86bb57ab0830 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 23 Sep 2019 17:48:51 +0800 Subject: [PATCH 01/10] add table name change ddl (#866) * Remove useless dependencies and add jsp-2.1-6.1.14 * Determine if principal is empty in getJdbcUrl method * fix bug when register worker but create master node in zookeeper * Remove useless dependencies such as hive-shims and log4j * upgrade commons-email from 1.2 to 1.5 and remove mail-1.4.5.jar * add support for postgresql in upgrade database * add support for postgresql in upgrade database * update mysql in upgrade from 1.1.0 to next version * add table name change ddl --- .../dao/upgrade/DolphinSchedulerManager.java | 13 +- .../dao/upgrade/MysqlUpgradeDao.java | 9 +- .../dao/upgrade/PostgresqlUpgradeDao.java | 21 +- .../cn/escheduler/dao/upgrade/UpgradeDao.java | 34 +- .../mysql/dolphinscheduler_ddl.sql | 486 ++++++++++++++++++ 5 files changed, 533 insertions(+), 30 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/DolphinSchedulerManager.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/DolphinSchedulerManager.java index 210b95ff6f..f15312a3f3 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/DolphinSchedulerManager.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/DolphinSchedulerManager.java @@ -81,7 +81,9 @@ public class DolphinSchedulerManager { String version = ""; // Gets the version of the current system if (upgradeDao.isExistsTable("t_escheduler_version")) { - version = upgradeDao.getCurrentVersion(); + version = upgradeDao.getCurrentVersion("t_escheduler_version"); + }else if(upgradeDao.isExistsTable("t_dolphinscheduler_version")){ + version = upgradeDao.getCurrentVersion("t_dolphinscheduler_version"); }else if(upgradeDao.isExistsColumn("t_escheduler_queue","create_time")){ version = "1.0.1"; }else if(upgradeDao.isExistsTable("t_escheduler_queue")){ @@ -93,21 +95,14 @@ public class DolphinSchedulerManager { // The target version of the upgrade String schemaVersion = ""; for(String schemaDir : schemaList) { - - schemaVersion = schemaDir.split("_")[0]; if(SchemaUtils.isAGreatVersion(schemaVersion , version)) { logger.info("upgrade DolphinScheduler metadata version from " + version + " to " + schemaVersion); - logger.info("Begin upgrading DolphinScheduler's table structure"); upgradeDao.upgradeDolphinScheduler(schemaDir); - if(SchemaUtils.isAGreatVersion(version,"1.0.1")){ - version = upgradeDao.getCurrentVersion(); - }else { - version = schemaVersion; - } + version = schemaVersion; } } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/MysqlUpgradeDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/MysqlUpgradeDao.java index 17832896bc..d6a8974d28 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/MysqlUpgradeDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/MysqlUpgradeDao.java @@ -28,8 +28,6 @@ import java.sql.SQLException; public class MysqlUpgradeDao extends UpgradeDao { public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); - private static final String T_VERSION_NAME = "t_escheduler_version"; - private static final String rootDir = System.getProperty("user.dir"); @Override protected void init() { @@ -54,10 +52,11 @@ public class MysqlUpgradeDao extends UpgradeDao { * @return */ public boolean isExistsTable(String tableName) { + ResultSet rs = null; Connection conn = null; try { conn = ConnectionFactory.getDataSource().getConnection(); - ResultSet rs = conn.getMetaData().getTables(null, null, tableName, null); + rs = conn.getMetaData().getTables(null, null, tableName, null); if (rs.next()) { return true; } else { @@ -68,8 +67,7 @@ public class MysqlUpgradeDao extends UpgradeDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(null, null, conn); - + ConnectionUtils.releaseResource(rs, null, conn); } } @@ -96,7 +94,6 @@ public class MysqlUpgradeDao extends UpgradeDao { throw new RuntimeException(e.getMessage(),e); } finally { ConnectionUtils.releaseResource(null, null, conn); - } } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/PostgresqlUpgradeDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/PostgresqlUpgradeDao.java index 03ec8c819a..87ac1b4bde 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/PostgresqlUpgradeDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/PostgresqlUpgradeDao.java @@ -29,8 +29,6 @@ import java.sql.SQLException; public class PostgresqlUpgradeDao extends UpgradeDao { public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); - private static final String T_VERSION_NAME = "t_escheduler_version"; - private static final String rootDir = System.getProperty("user.dir"); private static final String schema = getSchema(); @Override @@ -55,23 +53,24 @@ public class PostgresqlUpgradeDao extends UpgradeDao { super.initSchema(initSqlPath); } - private static String getSchema(){ + public static String getSchema(){ Connection conn = null; PreparedStatement pstmt = null; + ResultSet resultSet = null; try { conn = ConnectionFactory.getDataSource().getConnection(); pstmt = conn.prepareStatement("select current_schema()"); - ResultSet resultSet = pstmt.executeQuery(); + resultSet = pstmt.executeQuery(); while (resultSet.next()){ if(resultSet.isFirst()){ return resultSet.getString(1); } } + } catch (SQLException e) { logger.error(e.getMessage(),e); - } finally { - ConnectionUtils.releaseResource(null, null, conn); + ConnectionUtils.releaseResource(resultSet, pstmt, conn); } return ""; } @@ -83,10 +82,11 @@ public class PostgresqlUpgradeDao extends UpgradeDao { */ public boolean isExistsTable(String tableName) { Connection conn = null; + ResultSet rs = null; try { conn = ConnectionFactory.getDataSource().getConnection(); - ResultSet rs = conn.getMetaData().getTables(null, schema, tableName, null); + rs = conn.getMetaData().getTables(null, schema, tableName, null); if (rs.next()) { return true; } else { @@ -97,7 +97,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(null, null, conn); + ConnectionUtils.releaseResource(rs, null, conn); } } @@ -110,9 +110,10 @@ public class PostgresqlUpgradeDao extends UpgradeDao { */ public boolean isExistsColumn(String tableName,String columnName) { Connection conn = null; + ResultSet rs = null; try { conn = ConnectionFactory.getDataSource().getConnection(); - ResultSet rs = conn.getMetaData().getColumns(null,schema,tableName,columnName); + rs = conn.getMetaData().getColumns(null,schema,tableName,columnName); if (rs.next()) { return true; } else { @@ -123,7 +124,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(null, null, conn); + ConnectionUtils.releaseResource(rs, null, conn); } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java index 13c0deffb9..820d8a3c99 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java @@ -18,6 +18,7 @@ package cn.escheduler.dao.upgrade; import cn.escheduler.common.enums.DbType; import cn.escheduler.common.utils.ConnectionUtils; +import cn.escheduler.common.utils.SchemaUtils; import cn.escheduler.common.utils.ScriptRunner; import cn.escheduler.dao.AbstractBaseDao; import cn.escheduler.dao.datasource.ConnectionFactory; @@ -36,7 +37,9 @@ public abstract class UpgradeDao extends AbstractBaseDao { public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); private static final String T_VERSION_NAME = "t_escheduler_version"; + private static final String T_NEW_VERSION_NAME = "t_dolphinscheduler_version"; private static final String rootDir = System.getProperty("user.dir"); + private static final DbType dbType = getCurrentDbType(); @Override protected void init() { @@ -48,13 +51,24 @@ public abstract class UpgradeDao extends AbstractBaseDao { * @return */ public static DbType getDbType(){ + return dbType; + } + + /** + * get db type + * @return + */ + private static DbType getCurrentDbType(){ + Connection conn = null; try { - Connection conn = ConnectionFactory.getDataSource().getConnection(); + conn = ConnectionFactory.getDataSource().getConnection(); String name = conn.getMetaData().getDatabaseProductName().toUpperCase(); return DbType.valueOf(name); } catch (Exception e) { logger.error(e.getMessage(),e); return null; + }finally { + ConnectionUtils.releaseResource(null, null, conn); } } @@ -175,8 +189,8 @@ public abstract class UpgradeDao extends AbstractBaseDao { public abstract boolean isExistsColumn(String tableName,String columnName); - public String getCurrentVersion() { - String sql = String.format("select version from %s",T_VERSION_NAME); + public String getCurrentVersion(String versionName) { + String sql = String.format("select version from %s",versionName); Connection conn = null; ResultSet rs = null; PreparedStatement pstmt = null; @@ -231,6 +245,12 @@ public abstract class UpgradeDao extends AbstractBaseDao { pstmt = conn.prepareStatement(upgradeSQL); pstmt.setString(1, schemaVersion); pstmt.executeUpdate(); + }else if (isExistsTable(T_NEW_VERSION_NAME)) { + // Change version in the version table to the new version + String upgradeSQL = String.format("update %s set version = ?",T_NEW_VERSION_NAME); + pstmt = conn.prepareStatement(upgradeSQL); + pstmt.setString(1, schemaVersion); + pstmt.executeUpdate(); } conn.commit(); } catch (FileNotFoundException e) { @@ -275,7 +295,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { if (StringUtils.isEmpty(rootDir)) { throw new RuntimeException("Environment variable user.dir not found"); } - String mysqlSQLFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_dml.sql",rootDir,schemaDir,getDbType().name().toLowerCase()); + String mysqlSQLFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_ddl.sql",rootDir,schemaDir,getDbType().name().toLowerCase()); Connection conn = null; PreparedStatement pstmt = null; try { @@ -314,7 +334,11 @@ public abstract class UpgradeDao extends AbstractBaseDao { public void updateVersion(String version) { // Change version in the version table to the new version - String upgradeSQL = String.format("update %s set version = ?",T_VERSION_NAME); + String versionName = T_VERSION_NAME; + if(!SchemaUtils.isAGreatVersion("1.2.0" , version)){ + versionName = "t_dolphinscheduler_version"; + } + String upgradeSQL = String.format("update %s set version = ?",versionName); PreparedStatement pstmt = null; Connection conn = null; try { diff --git a/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_ddl.sql index e69de29bb2..5faeba33bc 100644 --- a/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,486 @@ +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); +-- ut_dolphin_T_t_dolphinscheduler_access_token +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_access_token; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_access_token() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_access_token' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_access_token RENAME t_dolphinscheduler_access_token; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_access_token; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_access_token; + +-- ut_dolphin_T_t_dolphinscheduler_alert +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_alert; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_alert() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_alert' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_alert RENAME t_dolphinscheduler_alert; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_alert; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_alert; + +-- ut_dolphin_T_t_dolphinscheduler_alertgroup +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_alertgroup; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_alertgroup() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_alertgroup' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_alertgroup RENAME t_dolphinscheduler_alertgroup; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_alertgroup; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_alertgroup; + +-- ut_dolphin_T_t_dolphinscheduler_command +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_command; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_command() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_command' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_command RENAME t_dolphinscheduler_command; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_command; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_command; + +-- ut_dolphin_T_t_dolphinscheduler_datasource +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_datasource; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_datasource() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_datasource' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_datasource RENAME t_dolphinscheduler_datasource; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_datasource; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_datasource; + +-- ut_dolphin_T_t_dolphinscheduler_error_command +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_error_command; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_error_command() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_error_command' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_error_command RENAME t_dolphinscheduler_error_command; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_error_command; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_error_command; + +-- ut_dolphin_T_t_dolphinscheduler_master_server +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_master_server; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_master_server() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_master_server' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_master_server RENAME t_dolphinscheduler_master_server; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_master_server; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_master_server; + +-- ut_dolphin_T_t_dolphinscheduler_process_definition +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_process_definition; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_process_definition() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_process_definition' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_process_definition RENAME t_dolphinscheduler_process_definition; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_process_definition; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_process_definition; + +-- ut_dolphin_T_t_dolphinscheduler_process_instance +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_process_instance; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_process_instance() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_process_instance' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_process_instance RENAME t_dolphinscheduler_process_instance; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_process_instance; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_process_instance; + +-- ut_dolphin_T_t_dolphinscheduler_project +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_project; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_project() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_project' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_project RENAME t_dolphinscheduler_project; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_project; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_project; + +-- ut_dolphin_T_t_dolphinscheduler_queue +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_queue; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_queue() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_queue' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_queue RENAME t_dolphinscheduler_queue; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_queue; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_queue; + +-- ut_dolphin_T_t_dolphinscheduler_relation_datasource_user +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_relation_datasource_user; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_datasource_user() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_relation_datasource_user' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_relation_datasource_user RENAME t_dolphinscheduler_relation_datasource_user; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_relation_datasource_user; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_datasource_user; + +-- ut_dolphin_T_t_dolphinscheduler_relation_process_instance +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_relation_process_instance; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_process_instance() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_relation_process_instance' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_relation_process_instance RENAME t_dolphinscheduler_relation_process_instance; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_relation_process_instance; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_process_instance; + +-- ut_dolphin_T_t_dolphinscheduler_relation_project_user +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_relation_project_user; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_project_user() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_relation_project_user' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_relation_project_user RENAME t_dolphinscheduler_relation_project_user; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_relation_project_user; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_project_user; + +-- ut_dolphin_T_t_dolphinscheduler_relation_resources_user +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_relation_resources_user; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_resources_user() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_relation_resources_user' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_relation_resources_user RENAME t_dolphinscheduler_relation_resources_user; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_relation_resources_user; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_resources_user; + +-- ut_dolphin_T_t_dolphinscheduler_relation_udfs_user +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_relation_udfs_user; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_udfs_user() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_relation_udfs_user' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_relation_udfs_user RENAME t_dolphinscheduler_relation_udfs_user; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_relation_udfs_user; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_udfs_user; + +-- ut_dolphin_T_t_dolphinscheduler_relation_user_alertgroup +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_relation_user_alertgroup; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_user_alertgroup() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_relation_user_alertgroup' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_relation_user_alertgroup RENAME t_dolphinscheduler_relation_user_alertgroup; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_relation_user_alertgroup; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_relation_user_alertgroup; + +-- ut_dolphin_T_t_dolphinscheduler_resources +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_resources; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_resources() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_resources' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_resources RENAME t_dolphinscheduler_resources; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_resources; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_resources; + +-- ut_dolphin_T_t_dolphinscheduler_schedules +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_schedules; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_schedules() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_schedules' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_schedules RENAME t_dolphinscheduler_schedules; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_schedules; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_schedules; + +-- ut_dolphin_T_t_dolphinscheduler_session +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_session; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_session() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_session' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_session RENAME t_dolphinscheduler_session; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_session; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_session; + +-- ut_dolphin_T_t_dolphinscheduler_task_instance +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_task_instance; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_task_instance() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_task_instance' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_task_instance RENAME t_dolphinscheduler_task_instance; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_task_instance; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_task_instance; + +-- ut_dolphin_T_t_dolphinscheduler_tenant +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_tenant; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_tenant() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_tenant' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_tenant RENAME t_dolphinscheduler_tenant; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_tenant; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_tenant; + +-- ut_dolphin_T_t_dolphinscheduler_udfs +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_udfs; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_udfs() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_udfs' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_udfs RENAME t_dolphinscheduler_udfs; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_udfs; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_udfs; + +-- ut_dolphin_T_t_dolphinscheduler_user +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_user; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_user() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_user' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_user RENAME t_dolphinscheduler_user; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_user; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_user; + +-- ut_dolphin_T_t_dolphinscheduler_version +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_version; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_version() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_version' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_version RENAME t_dolphinscheduler_version; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_version; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_version; + +-- ut_dolphin_T_t_dolphinscheduler_worker_group +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_worker_group; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_worker_group() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_worker_group' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_worker_group RENAME t_dolphinscheduler_worker_group; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_worker_group; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_worker_group; + +-- ut_dolphin_T_t_dolphinscheduler_worker_server +drop PROCEDURE if EXISTS ut_dolphin_T_t_dolphinscheduler_worker_server; +delimiter d// +CREATE PROCEDURE ut_dolphin_T_t_dolphinscheduler_worker_server() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.TABLES + WHERE TABLE_NAME='t_escheduler_worker_server' + AND TABLE_SCHEMA=(SELECT DATABASE())) + THEN + ALTER TABLE t_escheduler_worker_server RENAME t_dolphinscheduler_worker_server; + END IF; + END; +d// + +delimiter ; +CALL ut_dolphin_T_t_dolphinscheduler_worker_server; +DROP PROCEDURE ut_dolphin_T_t_dolphinscheduler_worker_server; \ No newline at end of file From 6525440c945ffc04880c3b3b313b00ebf2085b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=81=AA?= <389401879@qq.com> Date: Tue, 24 Sep 2019 15:09:24 +0800 Subject: [PATCH 02/10] =?UTF-8?q?[FEATURE][#841]Add=20HTTP=20components?= =?UTF-8?q?=EF=BC=88=E6=B7=BB=E5=8A=A0HTTP=E7=BB=84=E4=BB=B6=EF=BC=89=20(#?= =?UTF-8?q?842)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add HTTP components（添加HTTP组件） * HTTP组件的相关代码优化 * Repair HTTP Component Related Code Based on Optimized Recommendations (基于优化建议的http组件相关代码修复) * Repair HTTP connection leak problem (修复http连接泄漏问题) --- .../common/enums/HttpCheckCondition.java | 30 ++ .../escheduler/common/enums/HttpMethod.java | 31 ++ .../common/enums/HttpParametersType.java | 29 ++ .../cn/escheduler/common/enums/TaskType.java | 3 +- .../common/process/HttpProperty.java | 125 ++++++++ .../common/task/http/HttpParameters.java | 108 +++++++ .../common/utils/TaskParametersUtils.java | 3 + .../server/worker/task/TaskManager.java | 3 + .../server/worker/task/http/HttpTask.java | 270 ++++++++++++++++++ .../js/conf/home/pages/dag/_source/config.js | 4 + .../js/conf/home/pages/dag/_source/dag.scss | 3 + .../pages/dag/_source/formModel/formModel.vue | 8 + .../formModel/tasks/_source/commcon.js | 18 +- .../formModel/tasks/_source/httpParams.vue | 242 ++++++++++++++++ .../dag/_source/formModel/tasks/http.vue | 191 +++++++++++++ .../conf/home/pages/dag/img/toobar_HTTP.png | Bin 0 -> 386 bytes .../src/js/module/i18n/locale/en_US.js | 14 +- .../src/js/module/i18n/locale/zh_CN.js | 15 +- 18 files changed, 1093 insertions(+), 4 deletions(-) create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/enums/HttpCheckCondition.java create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/enums/HttpMethod.java create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/enums/HttpParametersType.java create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/process/HttpProperty.java create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/task/http/HttpParameters.java create mode 100644 escheduler-server/src/main/java/cn/escheduler/server/worker/task/http/HttpTask.java create mode 100644 escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/httpParams.vue create mode 100644 escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue create mode 100644 escheduler-ui/src/js/conf/home/pages/dag/img/toobar_HTTP.png diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/HttpCheckCondition.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/HttpCheckCondition.java new file mode 100644 index 0000000000..b88ec01980 --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/HttpCheckCondition.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.common.enums; + +/** + * http check condition + */ +public enum HttpCheckCondition { + /** + * 0 status_code_default:200 + * 1 status_code_custom + * 2 body_contains + * 3 body_not_contains + */ + STATUS_CODE_DEFAULT,STATUS_CODE_CUSTOM, BODY_CONTAINS, BODY_NOT_CONTAINS +} \ No newline at end of file diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/HttpMethod.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/HttpMethod.java new file mode 100644 index 0000000000..32cbf1c42f --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/HttpMethod.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.common.enums; + +/** + * http method + */ +public enum HttpMethod { + /** + * 0 get + * 1 post + * 2 head + * 3 put + * 4 delete + */ + GET, POST, HEAD, PUT, DELETE +} \ No newline at end of file diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/HttpParametersType.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/HttpParametersType.java new file mode 100644 index 0000000000..21f4d8b3cc --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/HttpParametersType.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.common.enums; + +/** + * http parameters type + */ +public enum HttpParametersType { + /** + * 0 parameter; + * 1 body; + * 2 headers; + */ + PARAMETER,BODY,HEADERS +} diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskType.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskType.java index 7e4fde6a34..a212261db6 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskType.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskType.java @@ -30,8 +30,9 @@ public enum TaskType { * 6 PYTHON * 7 DEPENDENT * 8 FLINK + * 9 HTTP */ - SHELL,SQL, SUB_PROCESS,PROCEDURE,MR,SPARK,PYTHON,DEPENDENT,FLINK; + SHELL,SQL, SUB_PROCESS,PROCEDURE,MR,SPARK,PYTHON,DEPENDENT,FLINK,HTTP; public static boolean typeIsNormalTask(String typeName) { TaskType taskType = TaskType.valueOf(typeName); diff --git a/escheduler-common/src/main/java/cn/escheduler/common/process/HttpProperty.java b/escheduler-common/src/main/java/cn/escheduler/common/process/HttpProperty.java new file mode 100644 index 0000000000..ed7cc46a38 --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/process/HttpProperty.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.common.process; + +import cn.escheduler.common.enums.HttpParametersType; + +import java.util.Objects; + +public class HttpProperty { + /** + * key + */ + private String prop; + + /** + * httpParametersType + */ + private HttpParametersType httpParametersType; + + /** + * value + */ + private String value; + + public HttpProperty() { + } + + public HttpProperty(String prop, HttpParametersType httpParametersType, String value) { + this.prop = prop; + this.httpParametersType = httpParametersType; + this.value = value; + } + + /** + * getter method + * + * @return the prop + * @see HttpProperty#prop + */ + public String getProp() { + return prop; + } + + /** + * setter method + * + * @param prop the prop to set + * @see HttpProperty#prop + */ + public void setProp(String prop) { + this.prop = prop; + } + + /** + * getter method + * + * @return the value + * @see HttpProperty#value + */ + public String getValue() { + return value; + } + + /** + * setter method + * + * @param value the value to set + * @see HttpProperty#value + */ + public void setValue(String value) { + this.value = value; + } + + public HttpParametersType getHttpParametersType() { + return httpParametersType; + } + + public void setHttpParametersType(HttpParametersType httpParametersType) { + this.httpParametersType = httpParametersType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + HttpProperty property = (HttpProperty) o; + return Objects.equals(prop, property.prop) && + Objects.equals(value, property.value); + } + + + @Override + public int hashCode() { + return Objects.hash(prop, value); + } + + @Override + public String toString() { + return "HttpProperty{" + + "prop='" + prop + '\'' + + ", httpParametersType=" + httpParametersType + + ", value='" + value + '\'' + + '}'; + } + + +} diff --git a/escheduler-common/src/main/java/cn/escheduler/common/task/http/HttpParameters.java b/escheduler-common/src/main/java/cn/escheduler/common/task/http/HttpParameters.java new file mode 100644 index 0000000000..dc23f738b8 --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/task/http/HttpParameters.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.common.task.http; + +import cn.escheduler.common.enums.HttpCheckCondition; +import cn.escheduler.common.enums.HttpMethod; +import cn.escheduler.common.process.HttpProperty; +import cn.escheduler.common.task.AbstractParameters; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * http parameter + */ +public class HttpParameters extends AbstractParameters { + /** + * url + */ + private String url; + + /** + * httpMethod + */ + private HttpMethod httpMethod; + + /** + * http params + */ + private List httpParams; + + /** + * httpCheckCondition + */ + private HttpCheckCondition httpCheckCondition = HttpCheckCondition.STATUS_CODE_DEFAULT; + + /** + * condition + */ + private String condition; + + + + @Override + public boolean checkParameters() { + return StringUtils.isNotEmpty(url); + } + + @Override + public List getResourceFilesList() { + return new ArrayList<>(); + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public HttpMethod getHttpMethod() { + return httpMethod; + } + + public void setHttpMethod(HttpMethod httpMethod) { + this.httpMethod = httpMethod; + } + + public List getHttpParams() { + return httpParams; + } + + public void setHttpParams(List httpParams) { + this.httpParams = httpParams; + } + + public HttpCheckCondition getHttpCheckCondition() { + return httpCheckCondition; + } + + public void setHttpCheckCondition(HttpCheckCondition httpCheckCondition) { + this.httpCheckCondition = httpCheckCondition; + } + + public String getCondition() { + return condition; + } + + public void setCondition(String condition) { + this.condition = condition; + } +} diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/TaskParametersUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/TaskParametersUtils.java index c8ceeb44a2..b3fcade4a7 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/TaskParametersUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/TaskParametersUtils.java @@ -20,6 +20,7 @@ import cn.escheduler.common.enums.TaskType; import cn.escheduler.common.task.AbstractParameters; import cn.escheduler.common.task.dependent.DependentParameters; import cn.escheduler.common.task.flink.FlinkParameters; +import cn.escheduler.common.task.http.HttpParameters; import cn.escheduler.common.task.mr.MapreduceParameters; import cn.escheduler.common.task.procedure.ProcedureParameters; import cn.escheduler.common.task.python.PythonParameters; @@ -66,6 +67,8 @@ public class TaskParametersUtils { return JSONUtils.parseObject(parameter, DependentParameters.class); case FLINK: return JSONUtils.parseObject(parameter, FlinkParameters.class); + case HTTP: + return JSONUtils.parseObject(parameter, HttpParameters.class); default: return null; } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/TaskManager.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/TaskManager.java index 986a6179c9..de96ff25be 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/TaskManager.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/TaskManager.java @@ -20,6 +20,7 @@ package cn.escheduler.server.worker.task; import cn.escheduler.common.enums.TaskType; import cn.escheduler.server.worker.task.dependent.DependentTask; import cn.escheduler.server.worker.task.flink.FlinkTask; +import cn.escheduler.server.worker.task.http.HttpTask; import cn.escheduler.server.worker.task.mr.MapReduceTask; import cn.escheduler.server.worker.task.processdure.ProcedureTask; import cn.escheduler.server.worker.task.python.PythonTask; @@ -62,6 +63,8 @@ public class TaskManager { return new PythonTask(props, logger); case DEPENDENT: return new DependentTask(props, logger); + case HTTP: + return new HttpTask(props, logger); default: logger.error("unsupport task type: {}", taskType); throw new IllegalArgumentException("not support task type"); diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/http/HttpTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/http/HttpTask.java new file mode 100644 index 0000000000..552e4bd828 --- /dev/null +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/http/HttpTask.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.server.worker.task.http; + + +import cn.escheduler.common.enums.HttpMethod; +import cn.escheduler.common.enums.HttpParametersType; +import cn.escheduler.common.process.HttpProperty; +import cn.escheduler.common.process.Property; +import cn.escheduler.common.task.AbstractParameters; +import cn.escheduler.common.task.http.HttpParameters; +import cn.escheduler.common.utils.Bytes; +import cn.escheduler.common.utils.DateUtils; +import cn.escheduler.common.utils.ParameterUtils; +import cn.escheduler.dao.DaoFactory; +import cn.escheduler.dao.ProcessDao; +import cn.escheduler.dao.model.ProcessInstance; +import cn.escheduler.server.utils.ParamUtils; +import cn.escheduler.server.worker.task.AbstractTask; +import cn.escheduler.server.worker.task.TaskProps; +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.io.Charsets; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.ParseException; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * http task + */ +public class HttpTask extends AbstractTask { + + private HttpParameters httpParameters; + + /** + * process database access + */ + private ProcessDao processDao; + + /** + * Convert mill seconds to second unit + */ + protected static final int MAX_CONNECTION_MILLISECONDS = 60000; + + protected static final String APPLICATION_JSON = "application/json"; + + protected String output; + + + public HttpTask(TaskProps props, Logger logger) { + super(props, logger); + this.processDao = DaoFactory.getDaoInstance(ProcessDao.class); + } + + @Override + public void init() { + logger.info("http task params {}", taskProps.getTaskParams()); + this.httpParameters = JSONObject.parseObject(taskProps.getTaskParams(), HttpParameters.class); + + if (!httpParameters.checkParameters()) { + throw new RuntimeException("http task params is not valid"); + } + } + + @Override + public void handle() throws Exception { + String threadLoggerInfoName = String.format("TaskLogInfo-%s", taskProps.getTaskAppId()); + Thread.currentThread().setName(threadLoggerInfoName); + + long startTime = System.currentTimeMillis(); + String statusCode = null; + String body = null; + try(CloseableHttpClient client = createHttpClient()) { + try(CloseableHttpResponse response = sendRequest(client)) { + statusCode = String.valueOf(getStatusCode(response)); + body = getResponseBody(response); + exitStatusCode = validResponse(body, statusCode); + long costTime = System.currentTimeMillis() - startTime; + logger.info("startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {}Millisecond, statusCode : {}, body : {}, log : {}", + DateUtils.format2Readable(startTime), httpParameters.getUrl(),httpParameters.getHttpMethod(), costTime, statusCode, body, output); + }catch (Exception e) { + appendMessage(e.toString()); + exitStatusCode = -1; + logger.error("httpUrl[" + httpParameters.getUrl() + "] connection failed："+output, e); + } + } catch (Exception e) { + appendMessage(e.toString()); + exitStatusCode = -1; + logger.error("httpUrl[" + httpParameters.getUrl() + "] connection failed："+output, e); + } + } + + protected CloseableHttpResponse sendRequest(CloseableHttpClient client) throws IOException { + RequestBuilder builder = createRequestBuilder(); + ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId()); + + Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), + taskProps.getDefinedParams(), + httpParameters.getLocalParametersMap(), + processInstance.getCmdTypeIfComplement(), + processInstance.getScheduleTime()); + List httpPropertyList = new ArrayList<>(); + if(httpParameters.getHttpParams() != null && httpParameters.getHttpParams().size() > 0){ + for (HttpProperty httpProperty: httpParameters.getHttpParams()) { + String jsonObject = JSONObject.toJSONString(httpProperty); + String params = ParameterUtils.convertParameterPlaceholders(jsonObject,ParamUtils.convert(paramsMap)); + logger.info("http request params：{}",params); + httpPropertyList.add(JSONObject.parseObject(params,HttpProperty.class)); + } + } + addRequestParams(builder,httpPropertyList); + HttpUriRequest request = builder.setUri(httpParameters.getUrl()).build(); + setHeaders(request,httpPropertyList); + return client.execute(request); + } + + protected String getResponseBody(CloseableHttpResponse httpResponse) throws ParseException, IOException { + if (httpResponse == null) { + return null; + } + HttpEntity entity = httpResponse.getEntity(); + if (entity == null) { + return null; + } + String webPage = EntityUtils.toString(entity, Bytes.UTF8_ENCODING); + return webPage; + } + + protected int getStatusCode(CloseableHttpResponse httpResponse) { + int status = httpResponse.getStatusLine().getStatusCode(); + return status; + } + + protected int validResponse(String body, String statusCode){ + int exitStatusCode = 0; + switch (httpParameters.getHttpCheckCondition()) { + case BODY_CONTAINS: + if (StringUtils.isEmpty(body) || !body.contains(httpParameters.getCondition())) { + appendMessage(httpParameters.getUrl() + " doesn contain " + + httpParameters.getCondition()); + exitStatusCode = -1; + } + break; + case BODY_NOT_CONTAINS: + if (StringUtils.isEmpty(body) || body.contains(httpParameters.getCondition())) { + appendMessage(httpParameters.getUrl() + " contains " + + httpParameters.getCondition()); + exitStatusCode = -1; + } + break; + case STATUS_CODE_CUSTOM: + if (!statusCode.equals(httpParameters.getCondition())) { + appendMessage(httpParameters.getUrl() + " statuscode: " + statusCode + ", Must be: " + httpParameters.getCondition()); + exitStatusCode = -1; + } + break; + default: + if (!"200".equals(statusCode)) { + appendMessage(httpParameters.getUrl() + " statuscode: " + statusCode + ", Must be: 200"); + exitStatusCode = -1; + } + break; + } + return exitStatusCode; + } + + public String getOutput() { + return output; + } + + protected void appendMessage(String message) { + if (output == null) { + output = ""; + } + if (message != null && !message.trim().isEmpty()) { + output += message; + } + } + + protected void addRequestParams(RequestBuilder builder,List httpPropertyList) { + if(httpPropertyList != null && httpPropertyList.size() > 0){ + JSONObject jsonParam = new JSONObject(); + for (HttpProperty property: httpPropertyList){ + if(property.getHttpParametersType() != null){ + if (property.getHttpParametersType().equals(HttpParametersType.PARAMETER)){ + builder.addParameter(property.getProp(), property.getValue()); + }else if(property.getHttpParametersType().equals(HttpParametersType.BODY)){ + jsonParam.put(property.getProp(), property.getValue()); + } + } + } + StringEntity postingString = new StringEntity(jsonParam.toString(), Charsets.UTF_8); + postingString.setContentEncoding(Bytes.UTF8_ENCODING); + postingString.setContentType(APPLICATION_JSON); + builder.setEntity(postingString); + } + } + + protected void setHeaders(HttpUriRequest request,List httpPropertyList) { + if(httpPropertyList != null && httpPropertyList.size() > 0){ + for (HttpProperty property: httpPropertyList){ + if(property.getHttpParametersType() != null) { + if (property.getHttpParametersType().equals(HttpParametersType.HEADERS)) { + request.addHeader(property.getProp(), property.getValue()); + } + } + } + } + } + + protected CloseableHttpClient createHttpClient() { + final RequestConfig requestConfig = requestConfig(); + HttpClientBuilder httpClientBuilder; + httpClientBuilder = HttpClients.custom().setDefaultRequestConfig(requestConfig); + return httpClientBuilder.build(); + } + + private RequestConfig requestConfig() { + return RequestConfig.custom().setSocketTimeout(MAX_CONNECTION_MILLISECONDS).setConnectTimeout(MAX_CONNECTION_MILLISECONDS).build(); + } + + protected RequestBuilder createRequestBuilder() { + if (httpParameters.getHttpMethod().equals(HttpMethod.GET)) { + return RequestBuilder.get(); + } else if (httpParameters.getHttpMethod().equals(HttpMethod.POST)) { + return RequestBuilder.post(); + } else if (httpParameters.getHttpMethod().equals(HttpMethod.HEAD)) { + return RequestBuilder.head(); + } else if (httpParameters.getHttpMethod().equals(HttpMethod.PUT)) { + return RequestBuilder.put(); + } else if (httpParameters.getHttpMethod().equals(HttpMethod.DELETE)) { + return RequestBuilder.delete(); + } else { + return null; + } + } + + @Override + public AbstractParameters getParameters() { + return this.httpParameters; + } +} diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js index 5305bf7476..54c0cdc5f5 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js @@ -275,6 +275,10 @@ let tasksType = { 'DEPENDENT': { desc: 'DEPENDENT', color: '#2FBFD8' + }, + 'HTTP': { + desc: 'HTTP', + color: '#E46F13' } } diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index 37d3acaa19..fefdbe670d 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -82,6 +82,9 @@ .icos-DEPENDENT { background: url("../img/toolbar_DEPENDENT.png") no-repeat 50% 50%; } + .icos-HTTP { + background: url("../img/toobar_HTTP.png") no-repeat 50% 50%; + } .toolbar { width: 60px; height: 100%; diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index a46b894d11..53da59643b 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -165,6 +165,12 @@ ref="DEPENDENT" :backfill-item="backfillItem"> + + @@ -189,6 +195,7 @@ import JSP from './../plugIn/jsPlumbHandle' import mProcedure from './tasks/procedure' import mDependent from './tasks/dependent' + import mHttp from './tasks/http' import mSubProcess from './tasks/sub_process' import mSelectInput from './_source/selectInput' import mTimeoutAlarm from './_source/timeoutAlarm' @@ -465,6 +472,7 @@ mFlink, mPython, mDependent, + mHttp, mSelectInput, mTimeoutAlarm, mPriority, diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js index 1897940064..1781455f8d 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js @@ -202,10 +202,26 @@ const sqlTypeList = [ } ] +const positionList = [ + { + id: 'PARAMETER', + code: "Parameter" + }, + { + id: 'BODY', + code: "Body" + }, + { + id: 'HEADERS', + code: "Headers" + } +] + export { cycleList, dateValueList, typeList, directList, - sqlTypeList + sqlTypeList, + positionList } diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/httpParams.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/httpParams.vue new file mode 100644 index 0000000000..343104f286 --- /dev/null +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/httpParams.vue @@ -0,0 +1,242 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + + + + diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue new file mode 100644 index 0000000000..3e40fce93c --- /dev/null +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue @@ -0,0 +1,191 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + diff --git a/escheduler-ui/src/js/conf/home/pages/dag/img/toobar_HTTP.png b/escheduler-ui/src/js/conf/home/pages/dag/img/toobar_HTTP.png new file mode 100644 index 0000000000000000000000000000000000000000..a0ca7bd141308aed80999648fc48ff6df6e4bba9 GIT binary patch literal 386 zcmV-|0e$|7P)ugAfF)o}yryf6l@yBrsN6+Y5xgg{6fYQ0DHaglY4X7bsQ@A1 z#$qWI*07>NUFNkWv;}k~+!)AWuqDVmlHwhI z5Xavm9pyNF4}8iNlQ4b-+$F~+U-nAa9(=FT&VnHxFr`520qpMy-~pa9CHLsPvh|j4 g`qS*c*Gqr_0KO>BDY%Gb6951J07*qoM6N<$g3O_tyZ`_I literal 0 HcmV?d00001 diff --git a/escheduler-ui/src/js/module/i18n/locale/en_US.js b/escheduler-ui/src/js/module/i18n/locale/en_US.js index e90e7fcf6d..2c866762e0 100644 --- a/escheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/escheduler-ui/src/js/module/i18n/locale/en_US.js @@ -196,7 +196,9 @@ export default { 'Non Query': 'Non Query', 'prop(required)': 'prop(required)', 'value(optional)': 'value(optional)', + 'value(required)': 'value(required)', 'prop is empty': 'prop is empty', + 'value is empty': 'value is empty', 'prop is repeat': 'prop is repeat', 'Start Time': 'Start Time', 'End Time': 'End Time', @@ -475,5 +477,15 @@ export default { 'warning of timeout': 'warning of timeout', 'Next five execution times': 'Next five execution times', 'Execute time': 'Execute time', - 'Complement range': 'Complement range' + 'Complement range': 'Complement range', + 'Http Url':'Http Url', + 'Http Method':'Http Method', + 'Http Parameters':'Http Parameters', + 'Http Parameters Key':'Http Parameters Key', + 'Http Parameters Position':'Http Parameters Position', + 'Http Parameters Value':'Http Parameters Value', + 'Http Check Condition':'Http Check Condition', + 'Http Condition':'Http Condition', + 'Please Enter Http Url': 'Please Enter Http Url(required)', + 'Please Enter Http Condition': 'Please Enter Http Condition' } diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index cfa0224185..1ccd9da08e 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -196,7 +196,9 @@ export default { 'Non Query': '非查询', 'prop(required)': 'prop(必填)', 'value(optional)': 'value(选填)', + 'value(required)': 'value(必填)', 'prop is empty': 'prop不能为空', + 'value is empty': 'value不能为空', 'prop is repeat': 'prop中有重复', 'Start Time': '开始时间', 'End Time': '结束时间', @@ -480,5 +482,16 @@ export default { 'slot':'slot数量', 'taskManager':'taskManage数量', 'jobManagerMemory':'jobManager内存数', - 'taskManagerMemory':'taskManager内存数' + 'taskManagerMemory':'taskManager内存数', + 'Complement range': '补数范围', + 'Http Url':'请求地址', + 'Http Method':'请求类型', + 'Http Parameters':'请求参数', + 'Http Parameters Key':'参数名', + 'Http Parameters Position':'参数位置', + 'Http Parameters Value':'参数值', + 'Http Check Condition':'校验条件', + 'Http Condition':'校验内容', + 'Please Enter Http Url': '请填写请求地址(必填)', + 'Please Enter Http Condition': '请填写校验内容' } From 5a13c4cb2d5a2b6ceafef4f852f20a2907b516a0 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 24 Sep 2019 16:43:13 +0800 Subject: [PATCH 03/10] add spring boot on alert server (#872) * Remove useless dependencies and add jsp-2.1-6.1.14 * Determine if principal is empty in getJdbcUrl method * fix bug when register worker but create master node in zookeeper * Remove useless dependencies such as hive-shims and log4j * upgrade commons-email from 1.2 to 1.5 and remove mail-1.4.5.jar * add support for postgresql in upgrade database * add support for postgresql in upgrade database * update mysql in upgrade from 1.1.0 to next version * add table name change ddl * add spring boot on alert server --- .../java/cn/escheduler/alert/AlertServer.java | 18 ++++++++++++++---- .../resources/application_alert.properties | 5 +++++ script/escheduler-daemon.sh | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 escheduler-alert/src/main/resources/application_alert.properties diff --git a/escheduler-alert/src/main/java/cn/escheduler/alert/AlertServer.java b/escheduler-alert/src/main/java/cn/escheduler/alert/AlertServer.java index 27c5f0ce56..b8402aacf1 100644 --- a/escheduler-alert/src/main/java/cn/escheduler/alert/AlertServer.java +++ b/escheduler-alert/src/main/java/cn/escheduler/alert/AlertServer.java @@ -20,28 +20,33 @@ import cn.escheduler.alert.runner.AlertSender; import cn.escheduler.alert.utils.Constants; import cn.escheduler.common.thread.Stopper; import cn.escheduler.dao.AlertDao; -import cn.escheduler.dao.DaoFactory; import cn.escheduler.dao.model.Alert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.context.annotation.ComponentScan; import java.util.List; /** * alert of start */ -public class AlertServer { +@ComponentScan("cn.escheduler") +public class AlertServer implements CommandLineRunner { private static final Logger logger = LoggerFactory.getLogger(AlertServer.class); /** * Alert Dao */ - private AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class); + @Autowired + private AlertDao alertDao; private AlertSender alertSender; private static volatile AlertServer instance; - private AlertServer() { + public AlertServer() { } @@ -72,7 +77,12 @@ public class AlertServer { public static void main(String[] args){ + SpringApplication app = new SpringApplication(AlertServer.class); + app.run(args); + } + @Override + public void run(String... strings) throws Exception { AlertServer alertServer = AlertServer.getInstance(); alertServer.start(); } diff --git a/escheduler-alert/src/main/resources/application_alert.properties b/escheduler-alert/src/main/resources/application_alert.properties new file mode 100644 index 0000000000..cbc81b06e6 --- /dev/null +++ b/escheduler-alert/src/main/resources/application_alert.properties @@ -0,0 +1,5 @@ +logging.config=classpath:alert_logback.xml + +# server port +server.port=7789 + diff --git a/script/escheduler-daemon.sh b/script/escheduler-daemon.sh index d54272c886..bb075f7aa8 100644 --- a/script/escheduler-daemon.sh +++ b/script/escheduler-daemon.sh @@ -50,7 +50,7 @@ elif [ "$command" = "worker-server" ]; then LOG_FILE="-Dspring.config.location=conf/application_worker.properties -Ddruid.mysql.usePingMethod=false" CLASS=cn.escheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then - LOG_FILE="-Dlogback.configurationFile=conf/alert_logback.xml" + LOG_FILE="-Dspring.config.location=conf/application_alert.properties -Ddruid.mysql.usePingMethod=false" CLASS=cn.escheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then CLASS=cn.escheduler.server.rpc.LoggerServer From 24740522e379a4a5e11774667046a791e2650135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=AB=A0=E7=AB=A0?= <2565307098@qq.com> Date: Wed, 25 Sep 2019 15:21:54 +0800 Subject: [PATCH 04/10] Update pom.xml (#876) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [BUG][#860] jar包冲突Correct the classpath of your application so that it contains a single, compatible version of javax.servlet.ServletContext --- escheduler-common/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index b5f1d58620..fe24d02209 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -572,6 +572,12 @@ org.mortbay.jetty jsp-2.1 + + + org.mortbay.jetty + servlet-api-2.5 + + From a58ed30167b5bb5b3993ad739fc52fd8c3e322d7 Mon Sep 17 00:00:00 2001 From: taox <490548661@qq.com> Date: Wed, 25 Sep 2019 18:46:37 +0800 Subject: [PATCH 05/10] Fix comment error in taskZkInfo function (#886) --- .../src/main/java/cn/escheduler/dao/ProcessDao.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 73ae868c80..dee22d6f36 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -1024,11 +1024,11 @@ public class ProcessDao extends AbstractBaseDao { } /** - * ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId}_${task executed by ip1},${ip2}... + * ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskInstanceId}_${task executed by ip1},${ip2}... * * The tasks with the highest priority are selected by comparing the priorities of the above four levels from high to low. * - * 流程实例优先级_流程实例id_任务优先级_任务id_任务执行机器ip1，ip2... high <- low + * 流程实例优先级_流程实例id_任务优先级_任务实例id_任务执行机器ip1，ip2... high <- low * * @param taskInstance * @return From c0912bd4686d718027b5c84d7b2a129838158c25 Mon Sep 17 00:00:00 2001 From: Yongchao Zhang <2565307098@qq.com> Date: Wed, 25 Sep 2019 21:23:22 +0800 Subject: [PATCH 06/10] update zh_CN.js delete Content repetition (#880) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit update zh_CN.js delete Content repetition . 'Complement range': '补数范围' --- escheduler-ui/src/js/module/i18n/locale/zh_CN.js | 1 - 1 file changed, 1 deletion(-) diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index 1ccd9da08e..2a0ca908e9 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -483,7 +483,6 @@ export default { 'taskManager':'taskManage数量', 'jobManagerMemory':'jobManager内存数', 'taskManagerMemory':'taskManager内存数', - 'Complement range': '补数范围', 'Http Url':'请求地址', 'Http Method':'请求类型', 'Http Parameters':'请求参数', From dde5cb21037e04a10a1ddf602228796452bf5a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=81=AA?= <389401879@qq.com> Date: Thu, 26 Sep 2019 10:27:04 +0800 Subject: [PATCH 07/10] [BUG][#888]repair the imported workflow timer is online and ineffective (#890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (导入工作流定时器状态为上线，不起效果) --- .../cn/escheduler/api/service/ProcessDefinitionService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java index 45d0b46e79..813e4b13fb 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java @@ -570,7 +570,7 @@ public class ProcessDefinitionService extends BaseDAGService { row.put("scheduleEndTime", schedule.getEndTime()); row.put("scheduleCrontab", schedule.getCrontab()); row.put("scheduleFailureStrategy", schedule.getFailureStrategy()); - row.put("scheduleReleaseState", schedule.getReleaseState()); + row.put("scheduleReleaseState", ReleaseState.OFFLINE); row.put("scheduleProcessInstancePriority", schedule.getProcessInstancePriority()); if(schedule.getId() == -1){ row.put("scheduleWorkerGroupId", -1); From b73c38a8b70723b38399628af642e58a0575e728 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 26 Sep 2019 11:15:54 +0800 Subject: [PATCH 08/10] Delete the'babel-plugin-transform-remove-console'dependency package (#873) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * package * 修改工作流实例页面状态为运行时，禁止删除，删除package.json包里面的babel-runtime和jasmine-core * bugfix-user-specified queue takes precedence over tenant queue (#769) Solve the problem that the user queue setting does not take effect * update markdown docs which can not display images normally (#806) * issue#728 (#746) The mailSender in PasswordAuthentication() and setFrom() may not be the same in actual use, and it's best to set it up separately. please add a example about this. * install-escheduler-ui.sh,monitor_server.py and install.sh scripts comment change to english and install-escheduler-ui.sh use escheduler change to dolphinscheduler (#812) * service start exception modify * master，worker start modify * .env update * install-escheduler-ui.sh,monitor_server.py and install.sh scripts comment change to english and install-escheduler-ui.sh use escheduler change to dolphinscheduler * scripts name standardization (#813) * service start exception modify * master，worker start modify * .env update * install-escheduler-ui.sh,monitor_server.py and install.sh scripts comment change to english and install-escheduler-ui.sh use escheduler change to dolphinscheduler * scripts name standardization * [BUG][#731]repair swagger annotation，interface path:/escheduler/projects/{projectName}/process/batch-delete and /escheduler/projects/{projectName}/process/delete (#764) * 增加工作流导出导入功能，前端定时器表达式的秒和分钟的默认值从*修改成0 * 修改工作流导出导入功能由excel改为json减少依赖 * [BUG][#731]repair swagger annotation ,interface path:/escheduler/projects/{projectName}/process/batch-delete and /escheduler/projects/{projectName}/process/delete * [BUG][#771] The edit timer did not assign the original data * [FEATURE][#236] Cross-project dependency * [FEATURE][#236] Cross-project dependency(增加跨项目依赖) * misspell words (#817) * (Docs): Fixed some typo errors (#811) * Update EasyScheduler Proposal.md * Update frontend-deployment.md * Update frontend-development.md * Update 前端开发文档.md * Update system-manual.md * Update HttpClientTest.java * Update 系统使用手册.md * fix singleton with volatile (#818) * Replace StringBuffer with StringBuilder inside the method (#816) * flink task support(flink 任务支持) (#711) * flink任务支持 * flink任务支持 * Update zh_CN.js * Update FlinkArgsUtils.java * Update .escheduler_env.sh * 1.Delete the page section console.log; 2.Change project name； 3.Production configuration packaging clears console.log * Delete the'babel-plugin-transform-remove-console'dependency package * 1.repair [BUG]#884,2.Add Scheduling Time Fields to Workflow Instance List，3.Add a reminder for empty data --- escheduler-ui/package.json | 1 - .../definition/pages/list/_source/timing.vue | 7 +++++- .../pages/index/_source/defineUserCount.vue | 6 +++-- .../instance/pages/list/_source/list.vue | 25 ++++++++++++------- .../pages/taskInstance/_source/list.vue | 2 +- .../src/js/module/i18n/locale/en_US.js | 4 ++- .../src/js/module/i18n/locale/zh_CN.js | 4 ++- .../src/lib/@analysys/ans-ui/package.json | 1 - 8 files changed, 33 insertions(+), 17 deletions(-) diff --git a/escheduler-ui/package.json b/escheduler-ui/package.json index 014205ba16..3a8d532e01 100644 --- a/escheduler-ui/package.json +++ b/escheduler-ui/package.json @@ -23,7 +23,6 @@ "babel-plugin-syntax-jsx": "^6.18.0", "babel-plugin-transform-class-properties": "^6.24.1", "babel-plugin-transform-object-rest-spread": "^6.26.0", - "babel-plugin-transform-remove-console": "^6.9.4", "babel-plugin-transform-runtime": "^6.23.0", "babel-plugin-transform-vue-jsx": "^3.5.0", "babel-preset-env": "^1.6.1", diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index 1dd6630c41..749529ce3f 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -253,7 +253,12 @@ let msg = '' this.store.dispatch(api, searchParams).then(res => { - this.previewTimes = res + if (res.length) { + this.previewTimes = res + } else { + this.$message.warning('该时间段无数据') + this.$message.warning(`${i18n.$t('There is no data for this period of time')}`) + } }) } }, diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/defineUserCount.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/defineUserCount.vue index 818765dae1..83eaec2e09 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/defineUserCount.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/defineUserCount.vue @@ -21,7 +21,8 @@ data () { return { isSpin: true, - msg: '' + msg: '', + parameter: {projectId: 0} } }, props: { @@ -54,7 +55,8 @@ }, created () { this.isSpin = true - this.getDefineUserCount(this.projectId).then(res => { + this.parameter.projectId = this.projectId; + this.getDefineUserCount(this.parameter).then(res => { this.defineUserList = [] this._handleDefineUser(res) this.isSpin = false diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index 9efc9f721a..09b39517f3 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -12,31 +12,34 @@ {{$t('Process Name')}} - + {{$t('Run Type')}} - + + {{$t('Scheduling Time')}} + + {{$t('Start Time')}} - + {{$t('End Time')}} - + {{$t('Duration')}}s - + {{$t('Run Times')}} {{$t('host')}} - + {{$t('fault-tolerant sign')}} - + {{$t('State')}} - + {{$t('Operation')}} @@ -46,9 +49,13 @@ {{parseInt(pageNo === 1 ? ($index + 1) : (($index + 1) + (pageSize * (pageNo - 1))))}} - {{item.name}} + {{item.name}} {{_rtRunningType(item.commandType)}} + + + {{item.scheduleTime | formatDate}} + {{item.startTime | formatDate}} {{item.endTime | formatDate}} diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue index c41828b39b..37e8563117 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue @@ -12,7 +12,7 @@ {{$t('Process Instance')}} - + {{$t('Node Type')}} diff --git a/escheduler-ui/src/js/module/i18n/locale/en_US.js b/escheduler-ui/src/js/module/i18n/locale/en_US.js index 2c866762e0..13477fec34 100644 --- a/escheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/escheduler-ui/src/js/module/i18n/locale/en_US.js @@ -232,6 +232,7 @@ export default { 'Service-Worker': 'Service-Worker', 'Process Name': 'Process Name', 'Run Type': 'Run Type', + 'Scheduling Time': 'Scheduling Time', 'Run Times': 'Run Times', 'host': 'host', 'fault-tolerant sign': 'fault-tolerant sign', @@ -487,5 +488,6 @@ export default { 'Http Check Condition':'Http Check Condition', 'Http Condition':'Http Condition', 'Please Enter Http Url': 'Please Enter Http Url(required)', - 'Please Enter Http Condition': 'Please Enter Http Condition' + 'Please Enter Http Condition': 'Please Enter Http Condition', + 'There is no data for this period of time': 'There is no data for this period of time' } diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index 2a0ca908e9..bdea1b827f 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -232,6 +232,7 @@ export default { 'Service-Worker': '服务管理-Worker', 'Process Name': '工作流名称', 'Run Type': '运行类型', + 'Scheduling Time': '调度时间', 'Run Times': '运行次数', 'host': 'host', 'fault-tolerant sign': '容错标识', @@ -492,5 +493,6 @@ export default { 'Http Check Condition':'校验条件', 'Http Condition':'校验内容', 'Please Enter Http Url': '请填写请求地址(必填)', - 'Please Enter Http Condition': '请填写校验内容' + 'Please Enter Http Condition': '请填写校验内容', + 'There is no data for this period of time': '该时间段无数据' } diff --git a/escheduler-ui/src/lib/@analysys/ans-ui/package.json b/escheduler-ui/src/lib/@analysys/ans-ui/package.json index 061f54a7ad..856dc6b243 100644 --- a/escheduler-ui/src/lib/@analysys/ans-ui/package.json +++ b/escheduler-ui/src/lib/@analysys/ans-ui/package.json @@ -49,7 +49,6 @@ "babel-plugin-syntax-jsx": "^6.18.0", "babel-plugin-transform-class-properties": "^6.24.1", "babel-plugin-transform-object-rest-spread": "^6.26.0", - "babel-plugin-transform-remove-console": "^6.9.4", "babel-plugin-transform-runtime": "^6.23.0", "babel-plugin-transform-vue-jsx": "^3.7.0", "babel-preset-env": "^1.5.2", From 6ff0d856b37f6b311dd51fa66a168c795d3be00b Mon Sep 17 00:00:00 2001 From: gary <408036296@163.com> Date: Thu, 26 Sep 2019 11:17:20 +0800 Subject: [PATCH 09/10] The class name with package name will be very long in large projects. It is recommended to increase the maximum length limit to 100(mysql t_escheduler_udfs.class_name is already varchar 255). (#892) --- .../resource/pages/udf/pages/function/_source/createUdf.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue index 37be2ebf22..feb7a86c17 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue @@ -27,7 +27,7 @@