Merge pull request #156 from lgcareer/dev-20190415

Dev 20190415:modify upgrade and create function

docs/zh_CN/后端部署文档.md

@@ -186,3 +186,7 @@ sh ./bin/escheduler-daemon.sh stop logger-server
 sh ./bin/escheduler-daemon.sh start alert-server
 sh ./bin/escheduler-daemon.sh stop alert-server
 ```
+
+## 3、数据库升级
+数据库升级是在1.0.2版本增加的功能,执行以下命令即可自动升级数据库。
+sh ./script/upgrade_escheduler.sh

escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/EschedulerManager.java

@@ -30,12 +30,17 @@ public class EschedulerManager {
     UpgradeDao upgradeDao = UpgradeDao.getInstance();
 
     public void initEscheduler() {
+        // Determines whether the escheduler table structure has been init
+        if(upgradeDao.isExistsTable("t_escheduler_version") || upgradeDao.isExistsTable("t_escheduler_queue")) {
+            logger.info("The database has been initialized. Skip the initialization step");
+            return;
+        }
         this.initEschedulerSchema();
     }
 
     public void initEschedulerSchema() {
 
-        logger.info("Start initializing the ark manager mysql table structure");
+        logger.info("Start initializing the escheduler manager mysql table structure");
         upgradeDao.initEschedulerSchema();
     }
 
@@ -52,15 +57,20 @@ public class EschedulerManager {
         }else {
 
             String version = "";
-            // The target version of the upgrade
-            String schemaVersion = "";
-            for(String schemaDir : schemaList) {
             // Gets the version of the current system
             if (upgradeDao.isExistsTable("t_escheduler_version")) {
                 version = upgradeDao.getCurrentVersion();
-                }else {
+            }else if(upgradeDao.isExistsColumn("t_escheduler_queue","create_time")){
+                version = "1.0.1";
+            }else if(upgradeDao.isExistsTable("t_escheduler_queue")){
                 version = "1.0.0";
+            }else{
+                logger.error("Unable to determine current software version, so cannot upgrade");
             }
+            // The target version of the upgrade
+            String schemaVersion = "";
+            for(String schemaDir : schemaList) {
+
 
                 schemaVersion = schemaDir.split("_")[0];
                 if(SchemaUtils.isAGreatVersion(schemaVersion , version)) {
@@ -70,7 +80,11 @@ public class EschedulerManager {
 
                     logger.info("Begin upgrading escheduler's mysql table structure");
                     upgradeDao.upgradeEscheduler(schemaDir);
-
+                    if(SchemaUtils.isAGreatVersion(version,"1.0.1")){
+                        version = upgradeDao.getCurrentVersion();
+                    }else {
+                        version = schemaVersion;
+                    }
                 }
 
             }

escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/UpgradeDao.java

@@ -20,6 +20,7 @@ import cn.escheduler.common.utils.MysqlUtil;
 import cn.escheduler.common.utils.ScriptRunner;
 import cn.escheduler.dao.AbstractBaseDao;
 import cn.escheduler.dao.datasource.ConnectionFactory;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,6 +34,7 @@ public class UpgradeDao extends AbstractBaseDao {
 
     public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class);
     private static final String T_VERSION_NAME = "t_escheduler_version";
+    private static final String rootDir = System.getProperty("user.dir");
 
     @Override
     protected void init() {
@@ -64,6 +66,10 @@ public class UpgradeDao extends AbstractBaseDao {
 
     private void runInitEschedulerDML() {
         Connection conn = null;
+        if (StringUtils.isEmpty(rootDir)) {
+            throw new RuntimeException("Environment variable user.dir not found");
+        }
+        String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql";
         try {
             conn = ConnectionFactory.getDataSource().getConnection();
             conn.setAutoCommit(false);
@@ -71,7 +77,7 @@ public class UpgradeDao extends AbstractBaseDao {
             // Execute the ark_manager_dml.sql script to import the data related to escheduler
 
             ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true);
-            Reader initSqlReader = new FileReader(new File("sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql"));
+            Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
             initScriptRunner.runScript(initSqlReader);
 
             conn.commit();
@@ -100,11 +106,15 @@ public class UpgradeDao extends AbstractBaseDao {
 
     private void runInitEschedulerDDL() {
         Connection conn = null;
+        if (StringUtils.isEmpty(rootDir)) {
+            throw new RuntimeException("Environment variable user.dir not found");
+        }
+        String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/escheduler_ddl.sql";
         try {
             conn = ConnectionFactory.getDataSource().getConnection();
             // Execute the escheduler_ddl.sql script to create the table structure of escheduler
             ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true);
-            Reader initSqlReader = new FileReader(new File("sql/create/release-1.0.0_schema/mysql/escheduler_ddl.sql"));
+            Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
             initScriptRunner.runScript(initSqlReader);
 
         } catch (IOException e) {
@@ -122,7 +132,11 @@ public class UpgradeDao extends AbstractBaseDao {
 
     }
 
-
+    /**
+     * Determines whether a table exists
+     * @param tableName
+     * @return
+     */
     public boolean isExistsTable(String tableName) {
         Connection conn = null;
         try {
@@ -144,6 +158,33 @@ public class UpgradeDao extends AbstractBaseDao {
 
     }
 
+    /**
+     * Determines whether a field exists in the specified table
+     * @param tableName
+     * @param columnName
+     * @return
+     */
+    public boolean isExistsColumn(String tableName,String columnName) {
+        Connection conn = null;
+        try {
+            conn = ConnectionFactory.getDataSource().getConnection();
+            ResultSet rs = conn.getMetaData().getColumns(null,null,tableName,columnName);
+            if (rs.next()) {
+                return true;
+            } else {
+                return false;
+            }
+
+        } catch (SQLException e) {
+            logger.error(e.getMessage(),e);
+            throw new RuntimeException(e.getMessage(),e);
+        } finally {
+            MysqlUtil.realeaseResource(null, null, conn);
+
+        }
+
+    }
+
 
     public String getCurrentVersion() {
         String sql = String.format("select version from %s",T_VERSION_NAME);
@@ -182,7 +223,10 @@ public class UpgradeDao extends AbstractBaseDao {
 
     private void upgradeEschedulerDML(String schemaDir) {
         String schemaVersion = schemaDir.split("_")[0];
-        String mysqlSQLFilePath = "sql/upgrade/" + schemaDir + "/mysql/escheduler_dml.sql";
+        if (StringUtils.isEmpty(rootDir)) {
+            throw new RuntimeException("Environment variable user.dir not found");
+        }
+        String mysqlSQLFilePath = rootDir + "/sql/upgrade/" + schemaDir + "/mysql/escheduler_dml.sql";
         Connection conn = null;
         PreparedStatement pstmt = null;
         try {
@@ -239,7 +283,10 @@ public class UpgradeDao extends AbstractBaseDao {
     }
 
     private void upgradeEschedulerDDL(String schemaDir) {
-        String mysqlSQLFilePath = "sql/upgrade/" + schemaDir + "/mysql/escheduler_ddl.sql";
+        if (StringUtils.isEmpty(rootDir)) {
+            throw new RuntimeException("Environment variable user.dir not found");
+        }
+        String mysqlSQLFilePath = rootDir + "/sql/upgrade/" + schemaDir + "/mysql/escheduler_ddl.sql";
         Connection conn = null;
         PreparedStatement pstmt = null;
         try {

escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java

@@ -29,7 +29,6 @@ public class CreateEscheduler {
 	private static final Logger logger = LoggerFactory.getLogger(CreateEscheduler.class);
 
 	public static void main(String[] args) {
-		Thread.currentThread().setName("manager-CreateEscheduler");
 		EschedulerManager eschedulerManager = new EschedulerManager();
 		eschedulerManager.initEscheduler();
 		logger.info("init escheduler finished");

script/create_escheduler.sh

@@ -1,6 +1,21 @@
 #!/bin/bash
-workDir=`dirname $0`
-workDir=`cd ${workDir};pwd`
-echo "$workDir/lib"
 
-java -Xmx1G -cp "$workDir/../lib/*"  cn.escheduler.dao.upgrade.shell.CreateEscheduler
+BIN_DIR=`dirname $0`
+BIN_DIR=`cd "$BIN_DIR"; pwd`
+ESCHEDULER_HOME=$BIN_DIR/..
+
+export JAVA_HOME=$JAVA_HOME
+
+
+export ESCHEDULER_CONF_DIR=$ESCHEDULER_HOME/conf
+export ESCHEDULER_LIB_JARS=$ESCHEDULER_HOME/lib/*
+
+export ESCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+export STOP_TIMEOUT=5
+
+CLASS=cn.escheduler.dao.upgrade.shell.CreateEscheduler
+
+exec_command="$ESCHEDULER_OPTS -classpath $ESCHEDULER_CONF_DIR:$ESCHEDULER_LIB_JARS $CLASS"
+
+cd $ESCHEDULER_HOME
+$JAVA_HOME/bin/java $exec_command

script/upgrade_escheduler.sh

@@ -1,6 +1,21 @@
 #!/bin/bash
-workDir=`dirname $0`
-workDir=`cd ${workDir};pwd`
-echo "$workDir/lib"
 
-java -Xmx1G -cp "$workDir/../lib/*"  cn.escheduler.dao.upgrade.shell.UpgradeEscheduler
+BIN_DIR=`dirname $0`
+BIN_DIR=`cd "$BIN_DIR"; pwd`
+ESCHEDULER_HOME=$BIN_DIR/..
+
+export JAVA_HOME=$JAVA_HOME
+
+
+export ESCHEDULER_CONF_DIR=$ESCHEDULER_HOME/conf
+export ESCHEDULER_LIB_JARS=$ESCHEDULER_HOME/lib/*
+
+export ESCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+export STOP_TIMEOUT=5
+
+CLASS=cn.escheduler.dao.upgrade.shell.UpgradeEscheduler
+
+exec_command="$ESCHEDULER_OPTS -classpath $ESCHEDULER_CONF_DIR:$ESCHEDULER_LIB_JARS $CLASS"
+
+cd $ESCHEDULER_HOME
+$JAVA_HOME/bin/java $exec_command

sql/upgrade/1.0.2_schema/mysql/escheduler_ddl.sql

@@ -181,3 +181,23 @@ d//
 delimiter ;
 CALL ac_escheduler_T_t_escheduler_schedules_C_worker_group_id;
 DROP PROCEDURE ac_escheduler_T_t_escheduler_schedules_C_worker_group_id;
+
+-- ac_escheduler_T_t_escheduler_process_instance_C_worker_group_id
+drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_process_instance_C_worker_group_id;
+delimiter d//
+CREATE PROCEDURE ac_escheduler_T_t_escheduler_process_instance_C_worker_group_id()
+   BEGIN
+       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
+           WHERE TABLE_NAME='t_escheduler_process_instance'
+           AND TABLE_SCHEMA=(SELECT DATABASE())
+           AND COLUMN_NAME='worker_group_id')
+   THEN
+         ALTER TABLE t_escheduler_process_instance ADD COLUMN `worker_group_id` int(11) NULL DEFAULT -1 COMMENT '任务指定运行的worker分组' AFTER `process_instance_priority`;
+       END IF;
+ END;
+
+d//
+
+delimiter ;
+CALL ac_escheduler_T_t_escheduler_process_instance_C_worker_group_id;
+DROP PROCEDURE ac_escheduler_T_t_escheduler_process_instance_C_worker_group_id;

sql/upgrade/1.0.2_schema/mysql/escheduler_dml.sql

@@ -1 +1 @@
-INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.0.0');
+INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.0.2');