From 283e7bd2ed8b8fcc7cb14f4ab48c2d7e81c16885 Mon Sep 17 00:00:00 2001 From: ligang Date: Thu, 25 Apr 2019 10:49:01 +0800 Subject: [PATCH 1/5] add upgrade --- docs/zh_CN/后端部署文档.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 86f1edeebc..6b105097af 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -185,4 +185,8 @@ sh ./bin/escheduler-daemon.sh stop logger-server ``` sh ./bin/escheduler-daemon.sh start alert-server sh ./bin/escheduler-daemon.sh stop alert-server -``` \ No newline at end of file +``` + +## 3、数据库升级 +数据库升级是在1.0.2版本增加的功能,执行以下命令即可自动升级数据库。 +sh ./script/upgrade_escheduler.sh \ No newline at end of file From b27f2c5c5ab36960d9fa6b97ac0623cf3aa97a39 Mon Sep 17 00:00:00 2001 From: ligang Date: Thu, 25 Apr 2019 18:22:39 +0800 Subject: [PATCH 2/5] update create and upgrade function --- .../dao/upgrade/EschedulerManager.java | 30 ++++++++++++----- .../cn/escheduler/dao/upgrade/UpgradeDao.java | 33 ++++++++++++++++++- .../dao/upgrade/shell/CreateEscheduler.java | 1 - .../1.0.2_schema/mysql/escheduler_dml.sql | 2 +- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/EschedulerManager.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/EschedulerManager.java index 34f07fbb29..0df98ec56e 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/EschedulerManager.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/EschedulerManager.java @@ -30,12 +30,17 @@ public class EschedulerManager { UpgradeDao upgradeDao = UpgradeDao.getInstance(); public void initEscheduler() { + // Determines whether the escheduler table structure has been init + if(upgradeDao.isExistsTable("t_escheduler_version") || upgradeDao.isExistsTable("t_escheduler_queue")) { + logger.info("The database has been initialized. Skip the initialization step"); + return; + } this.initEschedulerSchema(); } public void initEschedulerSchema() { - logger.info("Start initializing the ark manager mysql table structure"); + logger.info("Start initializing the escheduler manager mysql table structure"); upgradeDao.initEschedulerSchema(); } @@ -52,15 +57,20 @@ public class EschedulerManager { }else { String version = ""; + // Gets the version of the current system + if (upgradeDao.isExistsTable("t_escheduler_version")) { + version = upgradeDao.getCurrentVersion(); + }else if(upgradeDao.isExistsColumn("t_escheduler_queue","create_time")){ + version = "1.0.1"; + }else if(upgradeDao.isExistsTable("t_escheduler_queue")){ + version = "1.0.0"; + }else{ + logger.error("Unable to determine current software version, so cannot upgrade"); + } // The target version of the upgrade String schemaVersion = ""; for(String schemaDir : schemaList) { - // Gets the version of the current system - if (upgradeDao.isExistsTable("t_escheduler_version")) { - version = upgradeDao.getCurrentVersion(); - }else { - version = "1.0.0"; - } + schemaVersion = schemaDir.split("_")[0]; if(SchemaUtils.isAGreatVersion(schemaVersion , version)) { @@ -70,7 +80,11 @@ public class EschedulerManager { logger.info("Begin upgrading escheduler's mysql table structure"); upgradeDao.upgradeEscheduler(schemaDir); - + if(SchemaUtils.isAGreatVersion(version,"1.0.1")){ + version = upgradeDao.getCurrentVersion(); + }else { + version = schemaVersion; + } } } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java index f4fb307f05..21cc5af954 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java @@ -122,7 +122,11 @@ public class UpgradeDao extends AbstractBaseDao { } - + /** + * Determines whether a table exists + * @param tableName + * @return + */ public boolean isExistsTable(String tableName) { Connection conn = null; try { @@ -144,6 +148,33 @@ public class UpgradeDao extends AbstractBaseDao { } + /** + * Determines whether a field exists in the specified table + * @param tableName + * @param columnName + * @return + */ + public boolean isExistsColumn(String tableName,String columnName) { + Connection conn = null; + try { + conn = ConnectionFactory.getDataSource().getConnection(); + ResultSet rs = conn.getMetaData().getColumns(null,null,tableName,columnName); + if (rs.next()) { + return true; + } else { + return false; + } + + } catch (SQLException e) { + logger.error(e.getMessage(),e); + throw new RuntimeException(e.getMessage(),e); + } finally { + MysqlUtil.realeaseResource(null, null, conn); + + } + + } + public String getCurrentVersion() { String sql = String.format("select version from %s",T_VERSION_NAME); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java index 012c32bb9a..3726a70809 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java @@ -29,7 +29,6 @@ public class CreateEscheduler { private static final Logger logger = LoggerFactory.getLogger(CreateEscheduler.class); public static void main(String[] args) { - Thread.currentThread().setName("manager-CreateEscheduler"); EschedulerManager eschedulerManager = new EschedulerManager(); eschedulerManager.initEscheduler(); logger.info("init escheduler finished"); diff --git a/sql/upgrade/1.0.2_schema/mysql/escheduler_dml.sql b/sql/upgrade/1.0.2_schema/mysql/escheduler_dml.sql index b9c214b7ad..97da34884c 100644 --- a/sql/upgrade/1.0.2_schema/mysql/escheduler_dml.sql +++ b/sql/upgrade/1.0.2_schema/mysql/escheduler_dml.sql @@ -1 +1 @@ -INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.0.0'); \ No newline at end of file +INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.0.2'); \ No newline at end of file From 104d16058942b2a47d36c0baa5759423032c20cb Mon Sep 17 00:00:00 2001 From: ligang Date: Thu, 25 Apr 2019 18:59:00 +0800 Subject: [PATCH 3/5] add Environment variable user.dir --- .../cn/escheduler/dao/upgrade/UpgradeDao.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java index 21cc5af954..96199b0725 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java @@ -20,6 +20,7 @@ import cn.escheduler.common.utils.MysqlUtil; import cn.escheduler.common.utils.ScriptRunner; import cn.escheduler.dao.AbstractBaseDao; import cn.escheduler.dao.datasource.ConnectionFactory; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +34,7 @@ public class UpgradeDao extends AbstractBaseDao { public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); private static final String T_VERSION_NAME = "t_escheduler_version"; + private static final String rootDir = System.getProperty("user.dir"); @Override protected void init() { @@ -64,6 +66,10 @@ public class UpgradeDao extends AbstractBaseDao { private void runInitEschedulerDML() { Connection conn = null; + if (StringUtils.isEmpty(rootDir)) { + throw new RuntimeException("Environment variable user.dir not found"); + } + String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql"; try { conn = ConnectionFactory.getDataSource().getConnection(); conn.setAutoCommit(false); @@ -71,7 +77,7 @@ public class UpgradeDao extends AbstractBaseDao { // Execute the ark_manager_dml.sql script to import the data related to escheduler ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true); - Reader initSqlReader = new FileReader(new File("sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql")); + Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath)); initScriptRunner.runScript(initSqlReader); conn.commit(); @@ -100,11 +106,15 @@ public class UpgradeDao extends AbstractBaseDao { private void runInitEschedulerDDL() { Connection conn = null; + if (StringUtils.isEmpty(rootDir)) { + throw new RuntimeException("Environment variable user.dir not found"); + } + String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/escheduler_ddl.sql"; try { conn = ConnectionFactory.getDataSource().getConnection(); // Execute the escheduler_ddl.sql script to create the table structure of escheduler ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true); - Reader initSqlReader = new FileReader(new File("sql/create/release-1.0.0_schema/mysql/escheduler_ddl.sql")); + Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath)); initScriptRunner.runScript(initSqlReader); } catch (IOException e) { @@ -213,7 +223,10 @@ public class UpgradeDao extends AbstractBaseDao { private void upgradeEschedulerDML(String schemaDir) { String schemaVersion = schemaDir.split("_")[0]; - String mysqlSQLFilePath = "sql/upgrade/" + schemaDir + "/mysql/escheduler_dml.sql"; + if (StringUtils.isEmpty(rootDir)) { + throw new RuntimeException("Environment variable user.dir not found"); + } + String mysqlSQLFilePath = rootDir + "/sql/upgrade/" + schemaDir + "/mysql/escheduler_dml.sql"; Connection conn = null; PreparedStatement pstmt = null; try { @@ -270,7 +283,10 @@ public class UpgradeDao extends AbstractBaseDao { } private void upgradeEschedulerDDL(String schemaDir) { - String mysqlSQLFilePath = "sql/upgrade/" + schemaDir + "/mysql/escheduler_ddl.sql"; + if (StringUtils.isEmpty(rootDir)) { + throw new RuntimeException("Environment variable user.dir not found"); + } + String mysqlSQLFilePath = rootDir + "/sql/upgrade/" + schemaDir + "/mysql/escheduler_ddl.sql"; Connection conn = null; PreparedStatement pstmt = null; try { From f5f4a6b01b0739db7511815c885eb13374251319 Mon Sep 17 00:00:00 2001 From: ligang Date: Thu, 25 Apr 2019 19:04:09 +0800 Subject: [PATCH 4/5] update upgrade and create script --- script/create_escheduler.sh | 23 +++++++++++++++++++---- script/upgrade_escheduler.sh | 23 +++++++++++++++++++---- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/script/create_escheduler.sh b/script/create_escheduler.sh index dea3b07858..c88da7bb6d 100644 --- a/script/create_escheduler.sh +++ b/script/create_escheduler.sh @@ -1,6 +1,21 @@ #!/bin/bash -workDir=`dirname $0` -workDir=`cd ${workDir};pwd` -echo "$workDir/lib" -java -Xmx1G -cp "$workDir/../lib/*" cn.escheduler.dao.upgrade.shell.CreateEscheduler +BIN_DIR=`dirname $0` +BIN_DIR=`cd "$BIN_DIR"; pwd` +ESCHEDULER_HOME=$BIN_DIR/.. + +export JAVA_HOME=$JAVA_HOME + + +export ESCHEDULER_CONF_DIR=$ESCHEDULER_HOME/conf +export ESCHEDULER_LIB_JARS=$ESCHEDULER_HOME/lib/* + +export ESCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export STOP_TIMEOUT=5 + +CLASS=cn.escheduler.dao.upgrade.shell.CreateEscheduler + +exec_command="$ESCHEDULER_OPTS -classpath $ESCHEDULER_CONF_DIR:$ESCHEDULER_LIB_JARS $CLASS" + +cd $ESCHEDULER_HOME +$JAVA_HOME/bin/java $exec_command diff --git a/script/upgrade_escheduler.sh b/script/upgrade_escheduler.sh index 343dc5f1db..6bd6439a58 100644 --- a/script/upgrade_escheduler.sh +++ b/script/upgrade_escheduler.sh @@ -1,6 +1,21 @@ #!/bin/bash -workDir=`dirname $0` -workDir=`cd ${workDir};pwd` -echo "$workDir/lib" -java -Xmx1G -cp "$workDir/../lib/*" cn.escheduler.dao.upgrade.shell.UpgradeEscheduler +BIN_DIR=`dirname $0` +BIN_DIR=`cd "$BIN_DIR"; pwd` +ESCHEDULER_HOME=$BIN_DIR/.. + +export JAVA_HOME=$JAVA_HOME + + +export ESCHEDULER_CONF_DIR=$ESCHEDULER_HOME/conf +export ESCHEDULER_LIB_JARS=$ESCHEDULER_HOME/lib/* + +export ESCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export STOP_TIMEOUT=5 + +CLASS=cn.escheduler.dao.upgrade.shell.UpgradeEscheduler + +exec_command="$ESCHEDULER_OPTS -classpath $ESCHEDULER_CONF_DIR:$ESCHEDULER_LIB_JARS $CLASS" + +cd $ESCHEDULER_HOME +$JAVA_HOME/bin/java $exec_command From 90ca38a1c4a68a7201ecf6c82f2a480826139b38 Mon Sep 17 00:00:00 2001 From: ligang Date: Thu, 25 Apr 2019 19:20:25 +0800 Subject: [PATCH 5/5] update worker_group_id in t_escheduler_process_instance --- .../1.0.2_schema/mysql/escheduler_ddl.sql | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/sql/upgrade/1.0.2_schema/mysql/escheduler_ddl.sql b/sql/upgrade/1.0.2_schema/mysql/escheduler_ddl.sql index 2e3394fd41..435a8993a2 100644 --- a/sql/upgrade/1.0.2_schema/mysql/escheduler_ddl.sql +++ b/sql/upgrade/1.0.2_schema/mysql/escheduler_ddl.sql @@ -180,4 +180,24 @@ d// delimiter ; CALL ac_escheduler_T_t_escheduler_schedules_C_worker_group_id; -DROP PROCEDURE ac_escheduler_T_t_escheduler_schedules_C_worker_group_id; \ No newline at end of file +DROP PROCEDURE ac_escheduler_T_t_escheduler_schedules_C_worker_group_id; + +-- ac_escheduler_T_t_escheduler_process_instance_C_worker_group_id +drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_process_instance_C_worker_group_id; +delimiter d// +CREATE PROCEDURE ac_escheduler_T_t_escheduler_process_instance_C_worker_group_id() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_escheduler_process_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME='worker_group_id') + THEN + ALTER TABLE t_escheduler_process_instance ADD COLUMN `worker_group_id` int(11) NULL DEFAULT -1 COMMENT '任务指定运行的worker分组' AFTER `process_instance_priority`; + END IF; + END; + +d// + +delimiter ; +CALL ac_escheduler_T_t_escheduler_process_instance_C_worker_group_id; +DROP PROCEDURE ac_escheduler_T_t_escheduler_process_instance_C_worker_group_id; \ No newline at end of file