chore: Correct db upgrade and add ci avoid regression (#14620)

add atlasgo to ci
https://atlasgo.io/declarative/diff
related: https://github.com/apache/dolphinscheduler/issues/11286

BTW, it seems some ddl change only exist in 3.1.3、3.1.5、3.1.6 but do not exist in dev which will also cause an error when updating from those version

.github/workflows/backend.yml

@@ -42,6 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       not-ignore: ${{ steps.filter.outputs.not-ignore }}
+      db-schema: ${{ steps.filter.outputs.db-schema }}
     steps:
       - uses: actions/checkout@v2
       - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
@@ -50,6 +51,8 @@ jobs:
           filters: |
             not-ignore:
               - '!(docs/**)'
+            db-schema:
+              - 'dolphinscheduler-dao/src/main/resources/sql/**'
   build:
     name: Backend-Build
     needs: paths-filter
@@ -80,7 +83,7 @@ jobs:
       - name: Build and Package on ${{ matrix.java }}
         run: |
           ./mvnw -B clean install \
-                 -Prelease,docker \
+                 -Prelease \
                  -Dmaven.test.skip=true \
                  -Dspotless.skip=true \
                  -Dhttp.keepAlive=false \
@@ -120,20 +123,151 @@ jobs:
       - name: Running cluster test
         run: |
           /bin/bash ${{ matrix.case.script }}
+  schema-check:
+    runs-on: ubuntu-latest
+    if: ${{ (needs.paths-filter.outputs.db-schema == 'true') || (github.event_name == 'push') }}
+    timeout-minutes: 20
+    needs: build
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: mysql
+          MYSQL_DATABASE: dolphinscheduler_dev
+        ports:
+          - 3306:3306
+        options: --name=mysql --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dolphinscheduler_dev
+        ports:
+          - 5432:5432
+        options: --name=postgres --health-cmd=pg_isready --health-interval=10s --health-timeout=5s --health-retries=5
+    strategy:
+      fail-fast: false
+      matrix:
+        db: ["mysql", "postgresql"]
+        version: ["2.0.9", "3.0.6", "3.1.8"]
+    steps:
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: 8
+          distribution: 'adopt'
+      - name: Install Atlas and Create Dir
+        run: |
+          mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}
+          curl -sSf https://atlasgo.sh | sh
+      - name: Download Tarball
+        uses: actions/download-artifact@v2
+        with:
+          name: binary-package-8
+          path: dolphinscheduler/dev
+      - name: Set Env
+        run: |
+          VERSION=${{ matrix.version }}
+          echo "DATABASE_VERSION=${VERSION//\./}" >> $GITHUB_ENV
+      - name: Prepare
+        run: |
+          wget https://dlcdn.apache.org/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P dolphinscheduler/${{ matrix.version }}
+          tar -xzf dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -C dolphinscheduler/${{ matrix.version }} --strip-components 1
+          tar -xzf dolphinscheduler/dev/apache-dolphinscheduler-*-bin.tar.gz -C dolphinscheduler/dev --strip-components 1
+
+          if [[ ${{ matrix.db }} == "mysql" ]]; then
+            MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
+            MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar"
+            wget ${MYSQL_JDBC_URL} -O /tmp/${MYSQL_JDBC_JAR}
+            for base_dir in dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}; do
+              if [[ $base_dir == *"dolphinscheduler/2"* ]]; then
+                cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/lib
+              else
+                for d in alert-server api-server master-server worker-server tools; do
+                  cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs
+                done
+              fi
+            done
+            docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${{ env.DATABASE_VERSION }}"; 
+          else
+            docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${{ env.DATABASE_VERSION }};"
+          fi
+      - name: Check
+        run: |
+          if [[ $DATABASE_VERSION -lt 300 ]]; then
+            chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
+          else
+            chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
+          fi
+          if [[ ${{ matrix.db }} == "mysql" ]]; then
+            export DATABASE="mysql"
+            export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"
+            export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_dev?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
+            export SPRING_DATASOURCE_USERNAME="root"
+            export SPRING_DATASOURCE_PASSWORD="mysql"
+            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
+
+            export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
+            if [[ $DATABASE_VERSION -lt 300 ]]; then
+              bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
+            else
+              bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
+            fi
+            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
+
+            atlas_result=$(atlas schema diff \
+              --from "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}" \
+              --to "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_dev")
+            if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
+              echo "================================================================================================"
+              echo "                                !!!!! For Contributors !!!!!"
+              echo "================================================================================================"
+              echo "Database schema not sync, please add below change in the latest version of dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
+              echo "${atlas_result}"
+              exit 1
+            fi
+          else
+            export DATABASE="postgresql"
+            export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"
+            export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_dev"
+            export SPRING_DATASOURCE_USERNAME="postgres"
+            export SPRING_DATASOURCE_PASSWORD="postgres"
+            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
+          
+            export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}"
+            if [[ $DATABASE_VERSION -lt 300 ]]; then
+              bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
+            else
+              bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
+            fi
+            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
+          
+            atlas_result=$(atlas schema diff \
+              --from "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}?search_path=public&sslmode=disable" \
+              --to "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_dev?search_path=public&sslmode=disable")
+            if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
+              echo "================================================================================================"
+              echo "                                !!!!! For Contributors !!!!!"
+              echo "================================================================================================"
+              echo "Database schema not sync, please add below change in the latest version in dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
+              echo "${atlas_result}"
+              exit 1
+            fi
+          fi
   result:
     name: Build
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    needs: [ build, paths-filter, cluster-test ]
+    needs: [ build, paths-filter, cluster-test, schema-check ]
     if: always()
     steps:
       - name: Status
         run: |
-          if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
+          if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ needs.paths-filter.outputs.db-schema }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
             echo "Skip Build!"
             exit 0
           fi
-          if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then
+          if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' || ${{ needs.schema-check.result }} != 'success' ]]; then
             echo "Build Failed!"
             exit -1
           fi

dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql

@@ -1052,7 +1052,7 @@ CREATE TABLE t_ds_version
 -- Records of t_ds_version
 -- ----------------------------
 INSERT INTO t_ds_version
-VALUES ('1', 'dev');
+VALUES ('1', '3.3.0');
 
 
 -- ----------------------------

dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql

@@ -1043,7 +1043,7 @@ CREATE TABLE `t_ds_version` (
 -- ----------------------------
 -- Records of t_ds_version
 -- ----------------------------
-INSERT IGNORE INTO `t_ds_version` VALUES ('1', 'dev');
+INSERT IGNORE INTO `t_ds_version` VALUES ('1', '3.3.0');
 
 
 -- ----------------------------

dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql

@@ -1041,7 +1041,7 @@ INSERT INTO t_ds_queue(queue_name, queue, create_time, update_time)
 VALUES ('default', 'default', '2018-11-29 10:22:33', '2018-11-29 10:22:33');
 
 -- Records of t_ds_queue,default queue name : default
-INSERT INTO t_ds_version(version) VALUES ('dev');
+INSERT INTO t_ds_version(version) VALUES ('3.3.0');
 
 --
 -- Table structure for table t_ds_plugin_define

dolphinscheduler-dao/src/main/resources/sql/soft_version

@@ -1 +1 @@
-dev
+3.3.0

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.1.1_schema/postgresql/dolphinscheduler_ddl.sql

@@ -15,19 +15,8 @@
  * limitations under the License.
 */
 
---- rename t_ds_fav_task task_name to task_type
-DO $$
-DECLARE
-v_schema varchar;
-BEGIN
-    v_schema =current_schema();
-  IF EXISTS(SELECT *
-    FROM information_schema.columns
-    WHERE table_name='t_ds_fav_task' and column_name='task_name')
-  then
-   EXECUTE 'ALTER TABLE IF EXISTS ' || quote_ident(v_schema) ||'.t_ds_fav_task RENAME COLUMN task_name TO task_type';
-END IF;
-END $$;
+-- rename columns
+-- ALTER TABLE IF EXISTS t_ds_fav_task RENAME COLUMN task_name TO task_type;
 
---- add column
+-- add column
 ALTER TABLE t_ds_worker_group ADD COLUMN IF NOT EXISTS description varchar(255) DEFAULT NULL;

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql

@@ -16,6 +16,14 @@
 */
 SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));
 
+CREATE TABLE if not exists `t_ds_fav_task`
+(
+    `id`        bigint      NOT NULL AUTO_INCREMENT COMMENT 'id',
+    `task_type` varchar(64) NOT NULL COMMENT 'favorite task type name',
+    `user_id`   int         NOT NULL COMMENT 'user id',
+    PRIMARY KEY (`id`)
+) ENGINE = InnoDB AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8 COLLATE = utf8_bin;
+
 -- uc_dolphin_T_t_ds_command_R_test_flag
 drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_command_R_test_flag;
 delimiter d//
@@ -97,8 +105,7 @@ delimiter ;
 CALL uc_dolphin_T_t_ds_task_instance_R_test_flag;
 DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_R_test_flag;
 
-delimiter d//
-CREATE TABLE `t_ds_trigger_relation` (
+CREATE TABLE if not exists `t_ds_trigger_relation` (
     `id` bigint(20) NOT NULL AUTO_INCREMENT,
     `trigger_type` int(11) NOT NULL DEFAULT '0' COMMENT '0 process 1 task',
     `trigger_code` bigint(20) NOT NULL,
@@ -109,8 +116,6 @@ CREATE TABLE `t_ds_trigger_relation` (
     KEY `t_ds_trigger_relation_trigger_code_IDX` (`trigger_code`),
     UNIQUE KEY `t_ds_trigger_relation_UN` (`trigger_type`,`job_id`,`trigger_code`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
-d//
-delimiter ;
 
 -- uc_dolphin_T_t_ds_task_definition_R_is_cache
 drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_definition_R_is_cache;
@@ -182,7 +187,7 @@ BEGIN
            AND TABLE_SCHEMA=(SELECT DATABASE())
            AND COLUMN_NAME ='cache_key')
    THEN
-ALTER TABLE t_ds_task_instance ADD `cache_key` varchar(255) DEFAULT null COMMENT 'cache key';
+ALTER TABLE t_ds_task_instance ADD `cache_key` varchar(200) DEFAULT null COMMENT 'cache key';
 END IF;
 END;
 
@@ -202,7 +207,7 @@ BEGIN
         AND TABLE_SCHEMA=(SELECT DATABASE())
         AND INDEX_NAME='cache_key')
     THEN
-ALTER TABLE `t_ds_task_instance` ADD KEY `cache_key`( `cache_key` );
+ALTER TABLE `t_ds_task_instance` ADD KEY `idx_cache_key`( `cache_key` );
 END IF;
 END;
 d//
@@ -234,7 +239,7 @@ END IF;
            AND TABLE_SCHEMA=(SELECT DATABASE())
            AND COLUMN_NAME ='tenant_code')
    THEN
-ALTER TABLE t_ds_process_instance ADD `tenant_code` varchar(64) DEFAULT NULL COMMENT 'tenant code';
+ALTER TABLE t_ds_process_instance ADD `tenant_code` varchar(64) DEFAULT 'default' COMMENT 'tenant code';
 END IF;
 END;
 d//
@@ -356,6 +361,28 @@ ALTER TABLE `t_ds_task_group` MODIFY `name` varchar(255) DEFAULT NULL COMMENT 't
 ALTER TABLE `t_ds_k8s` MODIFY `k8s_name` varchar(255) DEFAULT NULL;
 ALTER TABLE `t_ds_k8s_namespace` MODIFY `namespace` varchar(255) DEFAULT NULL;
 ALTER TABLE `t_ds_cluster` MODIFY `name`        varchar(255) NOT NULL COMMENT 'cluster name';
+ALTER TABLE `t_ds_user` MODIFY COLUMN `tenant_id` int NULL DEFAULT -1 COMMENT 'tenant id';
+
+-- t_ds_fav_task behavior change
+DROP PROCEDURE if EXISTS add_improvement_t_ds_fav_task;
+delimiter d//
+CREATE PROCEDURE add_improvement_t_ds_fav_task()
+BEGIN
+   IF EXISTS (SELECT 1 FROM information_schema.COLUMNS
+           WHERE TABLE_NAME='t_ds_fav_task'
+           AND TABLE_SCHEMA=(SELECT DATABASE())
+           AND COLUMN_NAME ='task_name')
+   THEN
+       ALTER TABLE `t_ds_fav_task` MODIFY COLUMN `id` bigint NOT NULL AUTO_INCREMENT;
+       ALTER TABLE `t_ds_fav_task` DROP COLUMN `task_name`;
+       ALTER TABLE `t_ds_fav_task` ADD COLUMN `task_type` varchar(64) NOT NULL COMMENT 'favorite task type name';
+   END IF;
+END;
+d//
+delimiter ;
+CALL add_improvement_t_ds_fav_task;
+DROP PROCEDURE add_improvement_t_ds_fav_task;
+
 
 -- tenant improvement
 DROP PROCEDURE if EXISTS add_improvement_workflow_run_tenant;
@@ -389,16 +416,26 @@ delimiter ;
 CALL add_improvement_workflow_run_tenant;
 DROP PROCEDURE add_improvement_workflow_run_tenant;
 
--- uc_dolphin_T_t_ds_relation_sub_workflow
-drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_relation_sub_workflow;
+-- t_ds_fav_task behavior change
+drop PROCEDURE if EXISTS drop_t_ds_task_instance_key_foreign_key_instance_id;
 delimiter d//
-CREATE PROCEDURE uc_dolphin_T_t_ds_relation_sub_workflow()
+CREATE PROCEDURE drop_t_ds_task_instance_key_foreign_key_instance_id()
 BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-           WHERE TABLE_NAME='t_ds_relation_sub_workflow'
-           AND TABLE_SCHEMA=(SELECT DATABASE()))
-   THEN
-CREATE TABLE `t_ds_relation_sub_workflow` (
+    IF EXISTS (SELECT 1 FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS
+               WHERE TABLE_SCHEMA = (SELECT DATABASE())
+                 AND TABLE_NAME = 't_ds_task_instance'
+                 AND CONSTRAINT_NAME = 'foreign_key_instance_id')
+    THEN
+        ALTER TABLE `t_ds_task_instance` DROP FOREIGN KEY foreign_key_instance_id;
+    END IF;
+END;
+d//
+delimiter ;
+CALL drop_t_ds_task_instance_key_foreign_key_instance_id;
+DROP PROCEDURE drop_t_ds_task_instance_key_foreign_key_instance_id;
+
+-- table t_ds_relation_sub_workflow
+CREATE TABLE if not exists `t_ds_relation_sub_workflow` (
     `id` bigint NOT NULL AUTO_INCREMENT,
     `parent_workflow_instance_id` bigint  NOT NULL,
     `parent_task_code` bigint  NOT NULL,
@@ -408,7 +445,33 @@ CREATE TABLE `t_ds_relation_sub_workflow` (
     KEY `idx_parent_task_code` (`parent_task_code`),
     KEY `idx_sub_workflow_instance_id` (`sub_workflow_instance_id`)
 );
-END IF;
-END;
 
-d//
+-- table t_ds_project_preference
+CREATE TABLE if not exists `t_ds_project_preference` (
+   `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
+   `code` bigint(20) NOT NULL COMMENT 'encoding',
+   `project_code` bigint(20) NOT NULL COMMENT 'project code',
+   `preferences` varchar(512) NOT NULL COMMENT 'project preferences',
+   `user_id` int(11) DEFAULT NULL COMMENT 'creator id',
+   `state` int(11) DEFAULT '1' comment '1 means enabled, 0 means disabled',
+   `create_time` datetime NOT NULL COMMENT 'create time',
+   `update_time` datetime DEFAULT NULL COMMENT 'update time',
+   PRIMARY KEY (`id`),
+   UNIQUE KEY `unique_project_preference_project_code`(`project_code`),
+   UNIQUE KEY `unique_project_preference_code`(`code`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin;
+
+-- table t_ds_project_parameter
+CREATE TABLE if not exists `t_ds_project_parameter` (
+    `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
+    `param_name` varchar(255) NOT NULL COMMENT 'project parameter name',
+    `param_value` varchar(255) NOT NULL COMMENT 'project parameter value',
+    `code` bigint(20) NOT NULL COMMENT 'encoding',
+    `project_code` bigint(20) NOT NULL COMMENT 'project code',
+    `user_id` int(11) DEFAULT NULL COMMENT 'creator id',
+    `create_time` datetime NOT NULL COMMENT 'create time',
+    `update_time` datetime DEFAULT NULL COMMENT 'update time',
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `unique_project_parameter_name`(`project_code`, `param_name`),
+    UNIQUE KEY `unique_project_parameter_code`(`code`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin;

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl_post.sql

@@ -18,3 +18,27 @@
 ALTER TABLE t_ds_process_definition DROP tenant_id;
 ALTER TABLE t_ds_process_definition_log DROP tenant_id;
 ALTER TABLE t_ds_process_instance DROP tenant_id;
+
+SET FOREIGN_KEY_CHECKS = 0;
+-- auto detect by atlas, see more detail in https://github.com/apache/dolphinscheduler/pull/14620
+ALTER TABLE `QRTZ_BLOB_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL;
+ALTER TABLE `QRTZ_CALENDARS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `CALENDAR_NAME` varchar(200) NOT NULL;
+ALTER TABLE `QRTZ_CRON_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `CRON_EXPRESSION` varchar(120) NOT NULL, MODIFY COLUMN `TIME_ZONE_ID` varchar(80) NULL;
+ALTER TABLE `QRTZ_FIRED_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `ENTRY_ID` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `INSTANCE_NAME` varchar(200) NOT NULL, MODIFY COLUMN `STATE` varchar(16) NOT NULL, MODIFY COLUMN `JOB_NAME` varchar(200) NULL, MODIFY COLUMN `JOB_GROUP` varchar(200) NULL, MODIFY COLUMN `IS_NONCONCURRENT` varchar(1) NULL, MODIFY COLUMN `REQUESTS_RECOVERY` varchar(1) NULL;
+ALTER TABLE `QRTZ_JOB_DETAILS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `JOB_NAME` varchar(200) NOT NULL, MODIFY COLUMN `JOB_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `DESCRIPTION` varchar(250) NULL, MODIFY COLUMN `JOB_CLASS_NAME` varchar(250) NOT NULL, MODIFY COLUMN `IS_DURABLE` varchar(1) NOT NULL, MODIFY COLUMN `IS_NONCONCURRENT` varchar(1) NOT NULL, MODIFY COLUMN `IS_UPDATE_DATA` varchar(1) NOT NULL, MODIFY COLUMN `REQUESTS_RECOVERY` varchar(1) NOT NULL;
+ALTER TABLE `QRTZ_LOCKS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `LOCK_NAME` varchar(40) NOT NULL;
+ALTER TABLE `QRTZ_PAUSED_TRIGGER_GRPS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL;
+ALTER TABLE `QRTZ_SCHEDULER_STATE` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `INSTANCE_NAME` varchar(200) NOT NULL;
+ALTER TABLE `QRTZ_SIMPLE_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL;
+ALTER TABLE `QRTZ_SIMPROP_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `STR_PROP_1` varchar(512) NULL, MODIFY COLUMN `STR_PROP_2` varchar(512) NULL, MODIFY COLUMN `STR_PROP_3` varchar(512) NULL, MODIFY COLUMN `BOOL_PROP_1` varchar(1) NULL, MODIFY COLUMN `BOOL_PROP_2` varchar(1) NULL;
+ALTER TABLE `QRTZ_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `JOB_NAME` varchar(200) NOT NULL, MODIFY COLUMN `JOB_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `DESCRIPTION` varchar(250) NULL, MODIFY COLUMN `TRIGGER_STATE` varchar(16) NOT NULL, MODIFY COLUMN `TRIGGER_TYPE` varchar(8) NOT NULL, MODIFY COLUMN `CALENDAR_NAME` varchar(200) NULL;
+ALTER TABLE `t_ds_plugin_define` AUTO_INCREMENT 2;
+ALTER TABLE `t_ds_process_instance` MODIFY COLUMN `state_history` text NULL COMMENT 'state history desc';
+ALTER TABLE `t_ds_project` MODIFY COLUMN `description` varchar(255) NULL;
+ALTER TABLE `t_ds_task_group` MODIFY COLUMN `description` varchar(255) NULL;
+ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `app_link` text NULL COMMENT 'yarn app id', MODIFY COLUMN `cache_key` varchar(200) NULL COMMENT 'cache_key', MODIFY COLUMN `executor_name` varchar(64) NULL;
+ALTER TABLE `t_ds_worker_group` MODIFY COLUMN `description` text NULL COMMENT 'description';
+ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `cache_key` varchar(200) NULL COMMENT 'cache_key', MODIFY COLUMN `executor_name` varchar(64) NULL;
+ALTER TABLE `t_ds_fav_task` MODIFY COLUMN `id` bigint NOT NULL AUTO_INCREMENT COMMENT 'id';
+ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `cache_key` varchar(200) NULL COMMENT 'cache_key', MODIFY COLUMN `executor_name` varchar(64) NULL;
+SET FOREIGN_KEY_CHECKS = 1;

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql

@@ -14,97 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
 */
--- uc_dolphin_T_t_ds_command_R_test_flag
-delimiter ;
-DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_command_R_test_flag();
-delimiter d//
-CREATE FUNCTION uc_dolphin_T_t_ds_command_R_test_flag() RETURNS void AS $$
-BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_command'
-          AND COLUMN_NAME ='test_flag')
-      THEN
-ALTER TABLE t_ds_command alter column test_flag type int;
-ALTER TABLE t_ds_command alter column test_flag set DEFAULT NULL;
-END IF;
-END;
-$$ LANGUAGE plpgsql;
-d//
-delimiter ;
-select uc_dolphin_T_t_ds_command_R_test_flag();
-DROP FUNCTION uc_dolphin_T_t_ds_command_R_test_flag();
 
--- uc_dolphin_T_t_ds_error_command_R_test_flag
-delimiter ;
-DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_error_command_R_test_flag();
-delimiter d//
-CREATE FUNCTION uc_dolphin_T_t_ds_error_command_R_test_flag() RETURNS void AS $$
-BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_error_command'
-          AND COLUMN_NAME ='test_flag')
-      THEN
-ALTER TABLE t_ds_error_command alter column test_flag type int;
-ALTER TABLE t_ds_error_command alter column test_flag set DEFAULT NULL;
-END IF;
-END;
-$$ LANGUAGE plpgsql;
-d//
-delimiter ;
-select uc_dolphin_T_t_ds_error_command_R_test_flag();
-DROP FUNCTION uc_dolphin_T_t_ds_error_command_R_test_flag();
-
--- uc_dolphin_T_t_ds_process_instance_R_test_flag
-delimiter ;
-DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_instance_R_test_flag();
-delimiter d//
-CREATE FUNCTION uc_dolphin_T_t_ds_process_instance_R_test_flag() RETURNS void AS $$
-BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_process_instance'
-          AND COLUMN_NAME ='test_flag')
-      THEN
-ALTER TABLE t_ds_process_instance alter column test_flag type int;
-ALTER TABLE t_ds_process_instance alter column test_flag set DEFAULT NULL;
-END IF;
-END;
-$$ LANGUAGE plpgsql;
-d//
-delimiter ;
-select uc_dolphin_T_t_ds_process_instance_R_test_flag();
-DROP FUNCTION uc_dolphin_T_t_ds_process_instance_R_test_flag();
-
--- uc_dolphin_T_t_ds_task_instance_R_test_flag
-delimiter ;
-DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_R_test_flag();
-delimiter d//
-CREATE FUNCTION uc_dolphin_T_t_ds_task_instance_R_test_flag() RETURNS void AS $$
-BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_task_instance'
-          AND COLUMN_NAME ='test_flag')
-      THEN
-ALTER TABLE t_ds_task_instance alter column test_flag type int;
-ALTER TABLE t_ds_task_instance alter column test_flag set DEFAULT NULL;
-END IF;
-END;
-$$ LANGUAGE plpgsql;
-d//
-delimiter ;
-select uc_dolphin_T_t_ds_task_instance_R_test_flag();
-DROP FUNCTION uc_dolphin_T_t_ds_task_instance_R_test_flag();
-
-delimiter d//
-DROP TABLE IF EXISTS t_ds_trigger_relation;
-CREATE TABLE t_ds_trigger_relation (
+-- create table
+CREATE TABLE IF NOT EXISTS t_ds_trigger_relation (
     id        serial      NOT NULL,
     trigger_type int NOT NULL,
     trigger_code bigint NOT NULL,
@@ -114,204 +26,145 @@ CREATE TABLE t_ds_trigger_relation (
     PRIMARY KEY (id),
     CONSTRAINT t_ds_trigger_relation_unique UNIQUE (trigger_type,job_id,trigger_code)
 );
-d//
-delimiter ;
+CREATE TABLE IF NOT EXISTS t_ds_relation_sub_workflow (
+    id        serial      NOT NULL,
+    parent_workflow_instance_id BIGINT NOT NULL,
+    parent_task_code BIGINT NOT NULL,
+    sub_workflow_instance_id BIGINT NOT NULL,
+    PRIMARY KEY (id)
+);
+CREATE TABLE if not exists "t_ds_fav_task" (
+    "id" serial NOT NULL,
+    "task_type" VARCHAR(64) NOT NULL,
+    "user_id" integer NOT NULL,
+    PRIMARY KEY ("id")
+);
+CREATE TABLE if not exists "t_ds_project_preference" (
+    "id" int NOT NULL,
+    "code" bigint NOT NULL,
+    "project_code" bigint NOT NULL,
+    "preferences" VARCHAR(512) NOT NULL,
+    "user_id" integer NULL,
+    "state" integer NULL DEFAULT 1,
+    "create_time" timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    "update_time" timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY ("id")
+);
+CREATE TABLE if not exists "t_ds_project_parameter" (
+    "id" int NOT NULL,
+    "param_name" VARCHAR(255) NOT NULL,
+    "param_value" VARCHAR(255) NOT NULL,
+    "code" bigint NOT NULL,
+    "project_code" bigint NOT NULL,
+    "user_id" integer NULL,
+    "create_time" timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    "update_time" timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY ("id")
+);
 
-ALTER TABLE t_ds_task_definition DROP COLUMN IF EXISTS is_cache;
+-- add column, if you want to add constraint to the new column you should add them first
 ALTER TABLE t_ds_task_definition ADD COLUMN IF NOT EXISTS is_cache int DEFAULT '0';
-
-ALTER TABLE t_ds_task_definition_log DROP COLUMN IF EXISTS is_cache;
 ALTER TABLE t_ds_task_definition_log ADD COLUMN IF NOT EXISTS is_cache int DEFAULT '0';
-
-ALTER TABLE t_ds_task_instance DROP COLUMN IF EXISTS is_cache;
 ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS is_cache int DEFAULT '0';
-
 ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS cache_key varchar(200) DEFAULT NULL;
-ALTER TABLE t_ds_task_instance DROP COLUMN IF EXISTS cacke_key;
+ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS process_instance_name varchar(255);
+ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS executor_name varchar(64);
+ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS test_flag int;
+ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS project_code int;
+ALTER TABLE t_ds_process_instance ADD COLUMN IF NOT EXISTS project_code bigint;
+ALTER TABLE t_ds_process_instance ADD COLUMN IF NOT EXISTS executor_name varchar(64);
+ALTER TABLE t_ds_process_instance ADD COLUMN IF NOT EXISTS tenant_code varchar(64);
+ALTER TABLE t_ds_process_instance ADD COLUMN IF NOT EXISTS project_code bigint;
+ALTER TABLE t_ds_process_instance ADD COLUMN IF NOT EXISTS test_flag int;
+ALTER TABLE t_ds_command ADD COLUMN IF NOT EXISTS tenant_code varchar(64);
+ALTER TABLE t_ds_command ADD COLUMN IF NOT EXISTS test_flag int;
+ALTER TABLE t_ds_error_command ADD COLUMN IF NOT EXISTS tenant_code varchar(64);
+ALTER TABLE t_ds_error_command ADD COLUMN IF NOT EXISTS test_flag int;
+ALTER TABLE t_ds_schedules ADD COLUMN IF NOT EXISTS tenant_code varchar(64);
+ALTER TABLE t_ds_alert ADD COLUMN IF NOT EXISTS title varchar(512);
+ALTER TABLE t_ds_command ADD COLUMN IF NOT EXISTS worker_group varchar(255);
+ALTER TABLE t_ds_project ADD COLUMN IF NOT EXISTS name varchar(255);
+ALTER TABLE t_ds_schedules ADD COLUMN IF NOT EXISTS worker_group varchar(255);
+ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS worker_group varchar(255);
+ALTER TABLE t_ds_udfs ADD COLUMN IF NOT EXISTS func_name varchar(255);
+ALTER TABLE t_ds_version ADD COLUMN IF NOT EXISTS version varchar(63);
+ALTER TABLE t_ds_plugin_define ADD COLUMN IF NOT EXISTS plugin_name varchar(255);
+ALTER TABLE t_ds_plugin_define ADD COLUMN IF NOT EXISTS plugin_type varchar(63);
+ALTER TABLE t_ds_alert_plugin_instance ADD COLUMN IF NOT EXISTS instance_name varchar(255);
+ALTER TABLE t_ds_dq_rule ADD COLUMN IF NOT EXISTS name varchar(255);
+ALTER TABLE t_ds_environment ADD COLUMN IF NOT EXISTS name varchar(255);
+ALTER TABLE t_ds_task_group_queue ADD COLUMN IF NOT EXISTS task_name VARCHAR(255);
+ALTER TABLE t_ds_task_group ADD COLUMN IF NOT EXISTS name varchar(255);
+ALTER TABLE t_ds_k8s ADD COLUMN IF NOT EXISTS k8s_name VARCHAR(255);
+ALTER TABLE t_ds_k8s_namespace ADD COLUMN IF NOT EXISTS namespace varchar(255);
+ALTER TABLE t_ds_cluster ADD COLUMN IF NOT EXISTS name varchar(255);
+ALTER TABLE "t_ds_fav_task" ADD COLUMN IF NOT EXISTS "task_type" varchar(64) NOT NULL;
+
+-- alter column
+ALTER TABLE "t_ds_alert" ALTER COLUMN "title" TYPE VARCHAR(512);
+ALTER TABLE "t_ds_alert_plugin_instance" ALTER COLUMN "instance_name" TYPE VARCHAR(255);
+ALTER TABLE "t_ds_cluster" ALTER COLUMN "name" TYPE VARCHAR(255);
+ALTER TABLE "t_ds_command" ALTER COLUMN "worker_group" TYPE VARCHAR(255), ALTER COLUMN "tenant_code" SET DEFAULT 'default';
+ALTER TABLE "t_ds_dq_rule" ALTER COLUMN "name" TYPE VARCHAR(255);
+ALTER TABLE "t_ds_environment" ALTER COLUMN "name" TYPE VARCHAR(255);
+ALTER TABLE "t_ds_error_command" ALTER COLUMN "worker_group" TYPE VARCHAR(255), ALTER COLUMN "tenant_code" SET DEFAULT 'default';
+ALTER TABLE "t_ds_k8s" ALTER COLUMN "k8s_name" TYPE VARCHAR(255);
+ALTER TABLE "t_ds_k8s_namespace" ALTER COLUMN "namespace" TYPE VARCHAR(255), ALTER COLUMN "code" DROP DEFAULT, ALTER COLUMN "cluster_code" DROP DEFAULT;
+ALTER TABLE "t_ds_plugin_define" ALTER COLUMN "plugin_name" TYPE VARCHAR(255), ALTER COLUMN "plugin_type" TYPE VARCHAR(63);
+ALTER TABLE "t_ds_process_instance" ALTER COLUMN "worker_group" TYPE VARCHAR(255), ALTER COLUMN "executor_name" SET DEFAULT NULL::VARCHAR, ALTER COLUMN "tenant_code" SET DEFAULT 'default';
+ALTER TABLE "t_ds_project" ALTER COLUMN "name" TYPE VARCHAR(255);
+ALTER TABLE "t_ds_schedules" ALTER COLUMN "worker_group" TYPE VARCHAR(255), ALTER COLUMN "tenant_code" SET DEFAULT 'default';
+ALTER TABLE "t_ds_task_group" ALTER COLUMN "name" TYPE VARCHAR(255);
+ALTER TABLE "t_ds_task_group_queue" ALTER COLUMN "task_name" TYPE VARCHAR(255);
+ALTER TABLE "t_ds_task_instance" ALTER COLUMN "worker_group" TYPE VARCHAR(255), ALTER COLUMN "process_instance_name" SET DEFAULT NULL::VARCHAR, ALTER COLUMN "executor_name" SET DEFAULT NULL::VARCHAR, ALTER COLUMN "project_code" TYPE bigint;
+ALTER TABLE "t_ds_udfs" ALTER COLUMN "func_name" TYPE VARCHAR(255);
+ALTER TABLE "t_ds_user" ALTER COLUMN "tenant_id" SET DEFAULT -1;
+ALTER TABLE "t_ds_version" ALTER COLUMN "version" TYPE VARCHAR(63);
+ALTER TABLE "t_ds_worker_group" ALTER COLUMN "description" TYPE text, ALTER COLUMN "description" DROP DEFAULT;
+ALTER TABLE "t_ds_error_command" ALTER COLUMN "id" DROP DEFAULT;
+ALTER TABLE "t_ds_task_instance" ALTER COLUMN "log_path" DROP DEFAULT;
 
+-- create index
+DROP INDEX IF EXISTS idx_code;
+DROP INDEX IF EXISTS idx_process_code_version;
+DROP INDEX IF EXISTS idx_code_version;
+-- re index table t_ds_task_definition with index name task_definition_index
+DROP INDEX IF EXISTS task_definition_index;
 CREATE INDEX IF NOT EXISTS idx_cache_key ON t_ds_task_instance USING Btree("cache_key");
+CREATE INDEX IF NOT EXISTS idx_parent_workflow_instance_id ON t_ds_relation_sub_workflow (parent_workflow_instance_id);
+CREATE INDEX IF NOT EXISTS idx_parent_task_code ON t_ds_relation_sub_workflow (parent_task_code);
+CREATE INDEX IF NOT EXISTS idx_sub_workflow_instance_id ON t_ds_relation_sub_workflow (sub_workflow_instance_id);
+CREATE INDEX IF NOT EXISTS task_definition_index ON t_ds_task_definition (project_code, id);
+CREATE UNIQUE INDEX IF NOT EXISTS "unique_project_parameter_code" ON "t_ds_project_parameter" ("code");
+CREATE UNIQUE INDEX IF NOT EXISTS "unique_project_parameter_name" ON "t_ds_project_parameter" ("project_code", "param_name");
+CREATE UNIQUE INDEX IF NOT EXISTS "unique_project_preference_code" ON "t_ds_project_preference" ("code");
+CREATE UNIQUE INDEX IF NOT EXISTS "unique_project_preference_project_code" ON "t_ds_project_preference" ("project_code");
 
--- add_t_ds_process_instance_add_project_code
-delimiter ;
-DROP FUNCTION IF EXISTS add_t_ds_process_instance_add_project_code();
-delimiter d//
-CREATE FUNCTION add_t_ds_process_instance_add_project_code() RETURNS void AS $$
-BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_process_instance'
-          AND COLUMN_NAME ='project_code')
-      THEN
-ALTER TABLE t_ds_process_instance ADD project_code bigint;
-ALTER TABLE t_ds_process_instance alter column project_code set DEFAULT NULL;
+-- SEQUENCE
+DROP SEQUENCE IF EXISTS t_ds_project_preference_id_sequence;
+CREATE SEQUENCE t_ds_project_preference_id_sequence;
+ALTER TABLE t_ds_project_preference ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_project_preference_id_sequence');
+DROP SEQUENCE IF EXISTS t_ds_project_parameter_id_sequence;
+CREATE SEQUENCE  t_ds_project_parameter_id_sequence;
+ALTER TABLE t_ds_project_parameter ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_project_parameter_id_sequence');
+DROP SEQUENCE IF EXISTS "t_ds_error_command_id_seq";
+
+-- add comment
 comment on column t_ds_process_instance.project_code is 'project code';
-END IF;
-IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_process_instance'
-          AND COLUMN_NAME ='executor_name')
-      THEN
-ALTER TABLE t_ds_process_instance ADD executor_name varchar(64);
-ALTER TABLE t_ds_process_instance alter column executor_name set DEFAULT NULL;
 comment on column t_ds_process_instance.executor_name is 'execute user name';
-END IF;
-IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_process_instance'
-          AND COLUMN_NAME ='tenant_code')
-      THEN
-ALTER TABLE t_ds_process_instance ADD tenant_code varchar(64);
-ALTER TABLE t_ds_process_instance alter column tenant_code set DEFAULT NULL;
 comment on column t_ds_process_instance.tenant_code is 'tenant code';
-END IF;
-END;
-$$ LANGUAGE plpgsql;
-d//
-delimiter ;
-select add_t_ds_process_instance_add_project_code();
-DROP FUNCTION add_t_ds_process_instance_add_project_code();
-
--- add_t_ds_process_instance_add_project_code
-delimiter ;
-DROP FUNCTION IF EXISTS add_t_ds_task_instance_add_project_code();
-delimiter d//
-CREATE FUNCTION add_t_ds_task_instance_add_project_code() RETURNS void AS $$
-BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_task_instance'
-          AND COLUMN_NAME ='process_instance_name')
-      THEN
-ALTER TABLE t_ds_task_instance ADD process_instance_name varchar(255);
-ALTER TABLE t_ds_task_instance alter column process_instance_name set DEFAULT NULL;
 comment on column t_ds_task_instance.process_instance_name is 'process instance name';
-END IF;
-IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_task_instance'
-          AND COLUMN_NAME ='project_code')
-      THEN
-ALTER TABLE t_ds_process_instance ADD project_code bigint;
-ALTER TABLE t_ds_process_instance alter column project_code set DEFAULT NULL;
 comment on column t_ds_process_instance.project_code is 'project code';
-END IF;
-IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_task_instance'
-          AND COLUMN_NAME ='executor_name')
-      THEN
-ALTER TABLE t_ds_task_instance ADD executor_name varchar(64);
-ALTER TABLE t_ds_task_instance alter column executor_name set DEFAULT NULL;
 comment on column t_ds_task_instance.executor_name is 'execute user name';
-END IF;
-END;
-$$ LANGUAGE plpgsql;
-d//
-delimiter ;
-select add_t_ds_task_instance_add_project_code();
-DROP FUNCTION add_t_ds_task_instance_add_project_code();
-
-ALTER TABLE t_ds_alert alter column title type varchar(512);
-ALTER TABLE t_ds_alert alter column title set DEFAULT NULL;
-ALTER TABLE t_ds_command alter column worker_group type varchar(255);
-ALTER TABLE t_ds_project alter column name type varchar(255);
-ALTER TABLE t_ds_project alter column name set DEFAULT NULL;
-ALTER TABLE t_ds_schedules alter column worker_group type varchar(255);
-ALTER TABLE t_ds_task_instance alter column worker_group type varchar(255);
-ALTER TABLE t_ds_udfs alter column func_name type varchar(255);
-ALTER TABLE t_ds_udfs alter column func_name set NOT NULL ;
-ALTER TABLE t_ds_version alter column version type varchar(63);
-ALTER TABLE t_ds_version alter column version set NOT NULL;
-ALTER TABLE t_ds_plugin_define alter column plugin_name type varchar(255);
-ALTER TABLE t_ds_plugin_define alter column plugin_name set NOT NULL;
-ALTER TABLE t_ds_plugin_define alter column plugin_type type varchar(63);
-ALTER TABLE t_ds_plugin_define alter column plugin_type set NOT NULL;
-ALTER TABLE t_ds_alert_plugin_instance alter column instance_name type varchar(255);
-ALTER TABLE t_ds_alert_plugin_instance alter column instance_name set DEFAULT NULL;
-ALTER TABLE t_ds_dq_rule alter column name type varchar(255);
-ALTER TABLE t_ds_dq_rule alter column name set DEFAULT NULL;
-ALTER TABLE t_ds_environment alter column name type varchar(255);
-ALTER TABLE t_ds_environment alter column name set DEFAULT NULL;
-ALTER TABLE t_ds_task_group_queue alter column task_name type VARCHAR(255);
-ALTER TABLE t_ds_task_group_queue alter column task_name set DEFAULT NULL ;
-ALTER TABLE t_ds_task_group alter column name type varchar(255);
-ALTER TABLE t_ds_task_group alter column name set DEFAULT NULL ;
-ALTER TABLE t_ds_k8s alter column k8s_name type VARCHAR(255);
-ALTER TABLE t_ds_k8s alter column k8s_name set DEFAULT NULL ;
-ALTER TABLE t_ds_k8s_namespace alter column namespace type varchar(255);
-ALTER TABLE t_ds_k8s_namespace alter column namespace set DEFAULT NULL;
-ALTER TABLE t_ds_cluster alter column name type varchar(255);
-ALTER TABLE t_ds_cluster alter column name set DEFAULT NULL;
-
--- tenant improvement
-delimiter ;
-DROP FUNCTION IF EXISTS add_improvement_workflow_run_tenant();
-delimiter d//
-CREATE FUNCTION add_improvement_workflow_run_tenant() RETURNS void AS $$
-BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_command'
-          AND COLUMN_NAME ='tenant_code')
-      THEN
-ALTER TABLE t_ds_command ADD tenant_code varchar(64);
-ALTER TABLE t_ds_command alter column tenant_code set DEFAULT 'default';
 comment on column t_ds_command.tenant_code is 'tenant code';
-END IF;
-IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_error_command'
-          AND COLUMN_NAME ='tenant_code')
-      THEN
-ALTER TABLE t_ds_error_command ADD tenant_code varchar(64);
-ALTER TABLE t_ds_error_command alter column tenant_code set DEFAULT 'default';
 comment on column t_ds_error_command.tenant_code is 'tenant code';
-END IF;
-IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-          WHERE TABLE_CATALOG=current_database()
-          AND TABLE_SCHEMA=current_schema()
-          AND TABLE_NAME='t_ds_schedules'
-          AND COLUMN_NAME ='tenant_code')
-      THEN
-ALTER TABLE t_ds_schedules ADD tenant_code varchar(64);
-ALTER TABLE t_ds_schedules alter column tenant_code  set DEFAULT NULL;
 comment on column t_ds_schedules.tenant_code is 'tenant code';
-END IF;
-END;
-$$ LANGUAGE plpgsql;
-d//
-delimiter ;
-select add_improvement_workflow_run_tenant();
-DROP FUNCTION add_improvement_workflow_run_tenant();
-
--- uc_dolphin_T_t_ds_relation_sub_workflow
-CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_relation_sub_workflow()
-RETURNS VOID AS $$
-BEGIN
-    IF NOT EXISTS (
-        SELECT 1
-        FROM information_schema.columns
-        WHERE table_name='t_ds_relation_sub_workflow'
-        AND table_schema=current_schema()
-    ) THEN
-CREATE TABLE t_ds_relation_sub_workflow (
-                                            id        serial      NOT NULL,
-                                            parent_workflow_instance_id BIGINT NOT NULL,
-                                            parent_task_code BIGINT NOT NULL,
-                                            sub_workflow_instance_id BIGINT NOT NULL,
-                                            PRIMARY KEY (id)
-);
-CREATE INDEX idx_parent_workflow_instance_id ON t_ds_relation_sub_workflow (parent_workflow_instance_id);
-CREATE INDEX idx_parent_task_code ON t_ds_relation_sub_workflow (parent_task_code);
-CREATE INDEX idx_sub_workflow_instance_id ON t_ds_relation_sub_workflow (sub_workflow_instance_id);
-END IF;
-END;
-$$ LANGUAGE plpgsql;
+COMMENT ON COLUMN "t_ds_command" ."tenant_code" IS '';
+COMMENT ON COLUMN "t_ds_error_command" ."tenant_code" IS '';
+COMMENT ON COLUMN "t_ds_process_instance" ."project_code" IS '';
+COMMENT ON COLUMN "t_ds_process_instance" ."executor_name" IS '';
+COMMENT ON COLUMN "t_ds_process_instance" ."tenant_code" IS '';
+COMMENT ON COLUMN "t_ds_schedules" ."tenant_code" IS '';
+COMMENT ON COLUMN "t_ds_task_instance" ."process_instance_name" IS '';
+COMMENT ON COLUMN "t_ds_task_instance" ."executor_name" IS '';
+COMMENT ON COLUMN "t_ds_alert" ."sign" IS 'sign=sha1(content)';

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl_post.sql

@@ -15,6 +15,7 @@
  * limitations under the License.
 */
 
-ALTER TABLE "t_ds_process_definition" DROP "tenant_id";
-ALTER TABLE "t_ds_process_definition_log" DROP "tenant_id";
-ALTER TABLE "t_ds_process_instance" DROP "tenant_id";
+ALTER TABLE "t_ds_process_definition" DROP COLUMN IF EXISTS "tenant_id";
+ALTER TABLE "t_ds_process_definition_log" DROP COLUMN IF EXISTS "tenant_id";
+ALTER TABLE "t_ds_process_instance" DROP COLUMN IF EXISTS "tenant_id";
+ALTER TABLE "t_ds_fav_task" DROP COLUMN IF EXISTS "task_name";

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_dml.sql

@@ -18,7 +18,7 @@
 INSERT INTO t_ds_tenant(id, tenant_code, description, queue_id, create_time, update_time) VALUES (-1, 'default', 'default tenant', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22') ON CONFLICT (id) DO NOTHING;
 
 -- tenant improvement
-UPDATE t_ds_schedules t1 SET t1.tenant_code = COALESCE(t3.tenant_code, 'default') FROM t_ds_process_definition t2 LEFT JOIN t_ds_tenant t3 ON t2.tenant_id = t3.id WHERE t1.process_definition_code = t2.code;
+UPDATE t_ds_schedules as t1 SET tenant_code = COALESCE(t3.tenant_code, 'default') FROM t_ds_process_definition as t2 LEFT JOIN t_ds_tenant t3 ON t2.tenant_id = t3.id WHERE t1.process_definition_code = t2.code;
 UPDATE t_ds_process_instance SET tenant_code = 'default' WHERE tenant_code IS NULL;
 
 -- data quality support choose database

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql

@@ -0,0 +1,16 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl_post.sql

@@ -0,0 +1,16 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_dml.sql

@@ -0,0 +1,16 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql

@@ -0,0 +1,16 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl_post.sql

@@ -0,0 +1,16 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/

dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_dml.sql

@@ -0,0 +1,16 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/

dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java

@@ -27,10 +27,12 @@ import javax.sql.DataSource;
 
 import lombok.extern.slf4j.Slf4j;
 
+import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Service;
 
 @Service
 @Slf4j
+@Profile("mysql")
 public class MySQLUpgradeDao extends UpgradeDao {
 
     private MySQLUpgradeDao(DataSource dataSource) {

dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java

@@ -28,10 +28,12 @@ import javax.sql.DataSource;
 
 import lombok.extern.slf4j.Slf4j;
 
+import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Service;
 
 @Service
 @Slf4j
+@Profile("postgresql")
 public class PostgreSQLUpgradeDao extends UpgradeDao {
 
     private PostgreSQLUpgradeDao(DataSource dataSource) {