From 25e841a228e5c2ee30c269a911797825bc2fb9b4 Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Wed, 30 Aug 2023 11:55:05 +0800 Subject: [PATCH] chore: Correct db upgrade and add ci avoid regression (#14620) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add atlasgo to ci https://atlasgo.io/declarative/diff related: https://github.com/apache/dolphinscheduler/issues/11286 BTW, it seems some ddl change only exist in 3.1.3、3.1.5、3.1.6 but do not exist in dev which will also cause an error when updating from those version --- .github/workflows/backend.yml | 142 ++++++- .../resources/sql/dolphinscheduler_h2.sql | 2 +- .../resources/sql/dolphinscheduler_mysql.sql | 2 +- .../sql/dolphinscheduler_postgresql.sql | 2 +- .../src/main/resources/sql/soft_version | 2 +- .../postgresql/dolphinscheduler_ddl.sql | 17 +- .../mysql/dolphinscheduler_ddl.sql | 99 ++++- .../mysql/dolphinscheduler_ddl_post.sql | 24 ++ .../postgresql/dolphinscheduler_ddl.sql | 401 ++++++------------ .../postgresql/dolphinscheduler_ddl_post.sql | 7 +- .../postgresql/dolphinscheduler_dml.sql | 2 +- .../mysql/dolphinscheduler_ddl.sql | 16 + .../mysql/dolphinscheduler_ddl_post.sql | 16 + .../mysql/dolphinscheduler_dml.sql | 16 + .../postgresql/dolphinscheduler_ddl.sql | 16 + .../postgresql/dolphinscheduler_ddl_post.sql | 16 + .../postgresql/dolphinscheduler_dml.sql | 16 + .../tools/datasource/dao/MySQLUpgradeDao.java | 2 + .../datasource/dao/PostgreSQLUpgradeDao.java | 2 + 19 files changed, 482 insertions(+), 318 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl_post.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl_post.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_dml.sql diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 1240142f29..299b023366 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -42,6 +42,7 @@ jobs: runs-on: ubuntu-latest outputs: not-ignore: ${{ steps.filter.outputs.not-ignore }} + db-schema: ${{ steps.filter.outputs.db-schema }} steps: - uses: actions/checkout@v2 - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 @@ -50,6 +51,8 @@ jobs: filters: | not-ignore: - '!(docs/**)' + db-schema: + - 'dolphinscheduler-dao/src/main/resources/sql/**' build: name: Backend-Build needs: paths-filter @@ -80,7 +83,7 @@ jobs: - name: Build and Package on ${{ matrix.java }} run: | ./mvnw -B clean install \ - -Prelease,docker \ + -Prelease \ -Dmaven.test.skip=true \ -Dspotless.skip=true \ -Dhttp.keepAlive=false \ @@ -120,20 +123,151 @@ jobs: - name: Running cluster test run: | /bin/bash ${{ matrix.case.script }} + schema-check: + runs-on: ubuntu-latest + if: ${{ (needs.paths-filter.outputs.db-schema == 'true') || (github.event_name == 'push') }} + timeout-minutes: 20 + needs: build + services: + mysql: + image: mysql:5.7 + env: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: dolphinscheduler_dev + ports: + - 3306:3306 + options: --name=mysql --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dolphinscheduler_dev + ports: + - 5432:5432 + options: --name=postgres --health-cmd=pg_isready --health-interval=10s --health-timeout=5s --health-retries=5 + strategy: + fail-fast: false + matrix: + db: ["mysql", "postgresql"] + version: ["2.0.9", "3.0.6", "3.1.8"] + steps: + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: 8 + distribution: 'adopt' + - name: Install Atlas and Create Dir + run: | + mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }} + curl -sSf https://atlasgo.sh | sh + - name: Download Tarball + uses: actions/download-artifact@v2 + with: + name: binary-package-8 + path: dolphinscheduler/dev + - name: Set Env + run: | + VERSION=${{ matrix.version }} + echo "DATABASE_VERSION=${VERSION//\./}" >> $GITHUB_ENV + - name: Prepare + run: | + wget https://dlcdn.apache.org/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P dolphinscheduler/${{ matrix.version }} + tar -xzf dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -C dolphinscheduler/${{ matrix.version }} --strip-components 1 + tar -xzf dolphinscheduler/dev/apache-dolphinscheduler-*-bin.tar.gz -C dolphinscheduler/dev --strip-components 1 + + if [[ ${{ matrix.db }} == "mysql" ]]; then + MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" + MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar" + wget ${MYSQL_JDBC_URL} -O /tmp/${MYSQL_JDBC_JAR} + for base_dir in dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}; do + if [[ $base_dir == *"dolphinscheduler/2"* ]]; then + cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/lib + else + for d in alert-server api-server master-server worker-server tools; do + cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs + done + fi + done + docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${{ env.DATABASE_VERSION }}"; + else + docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${{ env.DATABASE_VERSION }};" + fi + - name: Check + run: | + if [[ $DATABASE_VERSION -lt 300 ]]; then + chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh + else + chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh + fi + if [[ ${{ matrix.db }} == "mysql" ]]; then + export DATABASE="mysql" + export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver" + export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_dev?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false" + export SPRING_DATASOURCE_USERNAME="root" + export SPRING_DATASOURCE_PASSWORD="mysql" + bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh + + export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false" + if [[ $DATABASE_VERSION -lt 300 ]]; then + bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh + else + bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh + fi + bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh + + atlas_result=$(atlas schema diff \ + --from "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}" \ + --to "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_dev") + if [[ ${atlas_result} != *"Schemas are synced"* ]]; then + echo "================================================================================================" + echo " !!!!! For Contributors !!!!!" + echo "================================================================================================" + echo "Database schema not sync, please add below change in the latest version of dolphinscheduler-dao/src/main/resources/sql/upgrade directory" + echo "${atlas_result}" + exit 1 + fi + else + export DATABASE="postgresql" + export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" + export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_dev" + export SPRING_DATASOURCE_USERNAME="postgres" + export SPRING_DATASOURCE_PASSWORD="postgres" + bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh + + export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}" + if [[ $DATABASE_VERSION -lt 300 ]]; then + bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh + else + bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh + fi + bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh + + atlas_result=$(atlas schema diff \ + --from "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}?search_path=public&sslmode=disable" \ + --to "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_dev?search_path=public&sslmode=disable") + if [[ ${atlas_result} != *"Schemas are synced"* ]]; then + echo "================================================================================================" + echo " !!!!! For Contributors !!!!!" + echo "================================================================================================" + echo "Database schema not sync, please add below change in the latest version in dolphinscheduler-dao/src/main/resources/sql/upgrade directory" + echo "${atlas_result}" + exit 1 + fi + fi result: name: Build runs-on: ubuntu-latest timeout-minutes: 30 - needs: [ build, paths-filter, cluster-test ] + needs: [ build, paths-filter, cluster-test, schema-check ] if: always() steps: - name: Status run: | - if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then + if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ needs.paths-filter.outputs.db-schema }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then echo "Skip Build!" exit 0 fi - if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then + if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' || ${{ needs.schema-check.result }} != 'success' ]]; then echo "Build Failed!" exit -1 fi diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 10e9d616ec..2604aa0e0a 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -1052,7 +1052,7 @@ CREATE TABLE t_ds_version -- Records of t_ds_version -- ---------------------------- INSERT INTO t_ds_version -VALUES ('1', 'dev'); +VALUES ('1', '3.3.0'); -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index 6078142c99..d6bfc3b2da 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -1043,7 +1043,7 @@ CREATE TABLE `t_ds_version` ( -- ---------------------------- -- Records of t_ds_version -- ---------------------------- -INSERT IGNORE INTO `t_ds_version` VALUES ('1', 'dev'); +INSERT IGNORE INTO `t_ds_version` VALUES ('1', '3.3.0'); -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 546ccf1574..bc006cfba3 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -1041,7 +1041,7 @@ INSERT INTO t_ds_queue(queue_name, queue, create_time, update_time) VALUES ('default', 'default', '2018-11-29 10:22:33', '2018-11-29 10:22:33'); -- Records of t_ds_queue,default queue name : default -INSERT INTO t_ds_version(version) VALUES ('dev'); +INSERT INTO t_ds_version(version) VALUES ('3.3.0'); -- -- Table structure for table t_ds_plugin_define diff --git a/dolphinscheduler-dao/src/main/resources/sql/soft_version b/dolphinscheduler-dao/src/main/resources/sql/soft_version index 38f8e886e1..15a2799817 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/soft_version +++ b/dolphinscheduler-dao/src/main/resources/sql/soft_version @@ -1 +1 @@ -dev +3.3.0 diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.1.1_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.1.1_schema/postgresql/dolphinscheduler_ddl.sql index b67b1247b9..79074b2077 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.1.1_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.1.1_schema/postgresql/dolphinscheduler_ddl.sql @@ -15,19 +15,8 @@ * limitations under the License. */ ---- rename t_ds_fav_task task_name to task_type -DO $$ -DECLARE -v_schema varchar; -BEGIN - v_schema =current_schema(); - IF EXISTS(SELECT * - FROM information_schema.columns - WHERE table_name='t_ds_fav_task' and column_name='task_name') - then - EXECUTE 'ALTER TABLE IF EXISTS ' || quote_ident(v_schema) ||'.t_ds_fav_task RENAME COLUMN task_name TO task_type'; -END IF; -END $$; +-- rename columns +-- ALTER TABLE IF EXISTS t_ds_fav_task RENAME COLUMN task_name TO task_type; ---- add column +-- add column ALTER TABLE t_ds_worker_group ADD COLUMN IF NOT EXISTS description varchar(255) DEFAULT NULL; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql index 3eca676c04..75a3ec2c3e 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql @@ -16,6 +16,14 @@ */ SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); +CREATE TABLE if not exists `t_ds_fav_task` +( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT 'id', + `task_type` varchar(64) NOT NULL COMMENT 'favorite task type name', + `user_id` int NOT NULL COMMENT 'user id', + PRIMARY KEY (`id`) +) ENGINE = InnoDB AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8 COLLATE = utf8_bin; + -- uc_dolphin_T_t_ds_command_R_test_flag drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_command_R_test_flag; delimiter d// @@ -97,8 +105,7 @@ delimiter ; CALL uc_dolphin_T_t_ds_task_instance_R_test_flag; DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_R_test_flag; -delimiter d// -CREATE TABLE `t_ds_trigger_relation` ( +CREATE TABLE if not exists `t_ds_trigger_relation` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `trigger_type` int(11) NOT NULL DEFAULT '0' COMMENT '0 process 1 task', `trigger_code` bigint(20) NOT NULL, @@ -109,8 +116,6 @@ CREATE TABLE `t_ds_trigger_relation` ( KEY `t_ds_trigger_relation_trigger_code_IDX` (`trigger_code`), UNIQUE KEY `t_ds_trigger_relation_UN` (`trigger_type`,`job_id`,`trigger_code`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; -d// -delimiter ; -- uc_dolphin_T_t_ds_task_definition_R_is_cache drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_definition_R_is_cache; @@ -182,7 +187,7 @@ BEGIN AND TABLE_SCHEMA=(SELECT DATABASE()) AND COLUMN_NAME ='cache_key') THEN -ALTER TABLE t_ds_task_instance ADD `cache_key` varchar(255) DEFAULT null COMMENT 'cache key'; +ALTER TABLE t_ds_task_instance ADD `cache_key` varchar(200) DEFAULT null COMMENT 'cache key'; END IF; END; @@ -202,7 +207,7 @@ BEGIN AND TABLE_SCHEMA=(SELECT DATABASE()) AND INDEX_NAME='cache_key') THEN -ALTER TABLE `t_ds_task_instance` ADD KEY `cache_key`( `cache_key` ); +ALTER TABLE `t_ds_task_instance` ADD KEY `idx_cache_key`( `cache_key` ); END IF; END; d// @@ -234,7 +239,7 @@ END IF; AND TABLE_SCHEMA=(SELECT DATABASE()) AND COLUMN_NAME ='tenant_code') THEN -ALTER TABLE t_ds_process_instance ADD `tenant_code` varchar(64) DEFAULT NULL COMMENT 'tenant code'; +ALTER TABLE t_ds_process_instance ADD `tenant_code` varchar(64) DEFAULT 'default' COMMENT 'tenant code'; END IF; END; d// @@ -356,6 +361,28 @@ ALTER TABLE `t_ds_task_group` MODIFY `name` varchar(255) DEFAULT NULL COMMENT 't ALTER TABLE `t_ds_k8s` MODIFY `k8s_name` varchar(255) DEFAULT NULL; ALTER TABLE `t_ds_k8s_namespace` MODIFY `namespace` varchar(255) DEFAULT NULL; ALTER TABLE `t_ds_cluster` MODIFY `name` varchar(255) NOT NULL COMMENT 'cluster name'; +ALTER TABLE `t_ds_user` MODIFY COLUMN `tenant_id` int NULL DEFAULT -1 COMMENT 'tenant id'; + +-- t_ds_fav_task behavior change +DROP PROCEDURE if EXISTS add_improvement_t_ds_fav_task; +delimiter d// +CREATE PROCEDURE add_improvement_t_ds_fav_task() +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_fav_task' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='task_name') + THEN + ALTER TABLE `t_ds_fav_task` MODIFY COLUMN `id` bigint NOT NULL AUTO_INCREMENT; + ALTER TABLE `t_ds_fav_task` DROP COLUMN `task_name`; + ALTER TABLE `t_ds_fav_task` ADD COLUMN `task_type` varchar(64) NOT NULL COMMENT 'favorite task type name'; + END IF; +END; +d// +delimiter ; +CALL add_improvement_t_ds_fav_task; +DROP PROCEDURE add_improvement_t_ds_fav_task; + -- tenant improvement DROP PROCEDURE if EXISTS add_improvement_workflow_run_tenant; @@ -389,16 +416,26 @@ delimiter ; CALL add_improvement_workflow_run_tenant; DROP PROCEDURE add_improvement_workflow_run_tenant; --- uc_dolphin_T_t_ds_relation_sub_workflow -drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_relation_sub_workflow; +-- t_ds_fav_task behavior change +drop PROCEDURE if EXISTS drop_t_ds_task_instance_key_foreign_key_instance_id; delimiter d// -CREATE PROCEDURE uc_dolphin_T_t_ds_relation_sub_workflow() +CREATE PROCEDURE drop_t_ds_task_instance_key_foreign_key_instance_id() BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_NAME='t_ds_relation_sub_workflow' - AND TABLE_SCHEMA=(SELECT DATABASE())) - THEN -CREATE TABLE `t_ds_relation_sub_workflow` ( + IF EXISTS (SELECT 1 FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS + WHERE TABLE_SCHEMA = (SELECT DATABASE()) + AND TABLE_NAME = 't_ds_task_instance' + AND CONSTRAINT_NAME = 'foreign_key_instance_id') + THEN + ALTER TABLE `t_ds_task_instance` DROP FOREIGN KEY foreign_key_instance_id; + END IF; +END; +d// +delimiter ; +CALL drop_t_ds_task_instance_key_foreign_key_instance_id; +DROP PROCEDURE drop_t_ds_task_instance_key_foreign_key_instance_id; + +-- table t_ds_relation_sub_workflow +CREATE TABLE if not exists `t_ds_relation_sub_workflow` ( `id` bigint NOT NULL AUTO_INCREMENT, `parent_workflow_instance_id` bigint NOT NULL, `parent_task_code` bigint NOT NULL, @@ -408,7 +445,33 @@ CREATE TABLE `t_ds_relation_sub_workflow` ( KEY `idx_parent_task_code` (`parent_task_code`), KEY `idx_sub_workflow_instance_id` (`sub_workflow_instance_id`) ); -END IF; -END; -d// +-- table t_ds_project_preference +CREATE TABLE if not exists `t_ds_project_preference` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key', + `code` bigint(20) NOT NULL COMMENT 'encoding', + `project_code` bigint(20) NOT NULL COMMENT 'project code', + `preferences` varchar(512) NOT NULL COMMENT 'project preferences', + `user_id` int(11) DEFAULT NULL COMMENT 'creator id', + `state` int(11) DEFAULT '1' comment '1 means enabled, 0 means disabled', + `create_time` datetime NOT NULL COMMENT 'create time', + `update_time` datetime DEFAULT NULL COMMENT 'update time', + PRIMARY KEY (`id`), + UNIQUE KEY `unique_project_preference_project_code`(`project_code`), + UNIQUE KEY `unique_project_preference_code`(`code`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; + +-- table t_ds_project_parameter +CREATE TABLE if not exists `t_ds_project_parameter` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key', + `param_name` varchar(255) NOT NULL COMMENT 'project parameter name', + `param_value` varchar(255) NOT NULL COMMENT 'project parameter value', + `code` bigint(20) NOT NULL COMMENT 'encoding', + `project_code` bigint(20) NOT NULL COMMENT 'project code', + `user_id` int(11) DEFAULT NULL COMMENT 'creator id', + `create_time` datetime NOT NULL COMMENT 'create time', + `update_time` datetime DEFAULT NULL COMMENT 'update time', + PRIMARY KEY (`id`), + UNIQUE KEY `unique_project_parameter_name`(`project_code`, `param_name`), + UNIQUE KEY `unique_project_parameter_code`(`code`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl_post.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl_post.sql index 82d05072f7..beadc63ca3 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl_post.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl_post.sql @@ -18,3 +18,27 @@ ALTER TABLE t_ds_process_definition DROP tenant_id; ALTER TABLE t_ds_process_definition_log DROP tenant_id; ALTER TABLE t_ds_process_instance DROP tenant_id; + +SET FOREIGN_KEY_CHECKS = 0; +-- auto detect by atlas, see more detail in https://github.com/apache/dolphinscheduler/pull/14620 +ALTER TABLE `QRTZ_BLOB_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL; +ALTER TABLE `QRTZ_CALENDARS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `CALENDAR_NAME` varchar(200) NOT NULL; +ALTER TABLE `QRTZ_CRON_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `CRON_EXPRESSION` varchar(120) NOT NULL, MODIFY COLUMN `TIME_ZONE_ID` varchar(80) NULL; +ALTER TABLE `QRTZ_FIRED_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `ENTRY_ID` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `INSTANCE_NAME` varchar(200) NOT NULL, MODIFY COLUMN `STATE` varchar(16) NOT NULL, MODIFY COLUMN `JOB_NAME` varchar(200) NULL, MODIFY COLUMN `JOB_GROUP` varchar(200) NULL, MODIFY COLUMN `IS_NONCONCURRENT` varchar(1) NULL, MODIFY COLUMN `REQUESTS_RECOVERY` varchar(1) NULL; +ALTER TABLE `QRTZ_JOB_DETAILS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `JOB_NAME` varchar(200) NOT NULL, MODIFY COLUMN `JOB_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `DESCRIPTION` varchar(250) NULL, MODIFY COLUMN `JOB_CLASS_NAME` varchar(250) NOT NULL, MODIFY COLUMN `IS_DURABLE` varchar(1) NOT NULL, MODIFY COLUMN `IS_NONCONCURRENT` varchar(1) NOT NULL, MODIFY COLUMN `IS_UPDATE_DATA` varchar(1) NOT NULL, MODIFY COLUMN `REQUESTS_RECOVERY` varchar(1) NOT NULL; +ALTER TABLE `QRTZ_LOCKS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `LOCK_NAME` varchar(40) NOT NULL; +ALTER TABLE `QRTZ_PAUSED_TRIGGER_GRPS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL; +ALTER TABLE `QRTZ_SCHEDULER_STATE` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `INSTANCE_NAME` varchar(200) NOT NULL; +ALTER TABLE `QRTZ_SIMPLE_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL; +ALTER TABLE `QRTZ_SIMPROP_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `STR_PROP_1` varchar(512) NULL, MODIFY COLUMN `STR_PROP_2` varchar(512) NULL, MODIFY COLUMN `STR_PROP_3` varchar(512) NULL, MODIFY COLUMN `BOOL_PROP_1` varchar(1) NULL, MODIFY COLUMN `BOOL_PROP_2` varchar(1) NULL; +ALTER TABLE `QRTZ_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `JOB_NAME` varchar(200) NOT NULL, MODIFY COLUMN `JOB_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `DESCRIPTION` varchar(250) NULL, MODIFY COLUMN `TRIGGER_STATE` varchar(16) NOT NULL, MODIFY COLUMN `TRIGGER_TYPE` varchar(8) NOT NULL, MODIFY COLUMN `CALENDAR_NAME` varchar(200) NULL; +ALTER TABLE `t_ds_plugin_define` AUTO_INCREMENT 2; +ALTER TABLE `t_ds_process_instance` MODIFY COLUMN `state_history` text NULL COMMENT 'state history desc'; +ALTER TABLE `t_ds_project` MODIFY COLUMN `description` varchar(255) NULL; +ALTER TABLE `t_ds_task_group` MODIFY COLUMN `description` varchar(255) NULL; +ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `app_link` text NULL COMMENT 'yarn app id', MODIFY COLUMN `cache_key` varchar(200) NULL COMMENT 'cache_key', MODIFY COLUMN `executor_name` varchar(64) NULL; +ALTER TABLE `t_ds_worker_group` MODIFY COLUMN `description` text NULL COMMENT 'description'; +ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `cache_key` varchar(200) NULL COMMENT 'cache_key', MODIFY COLUMN `executor_name` varchar(64) NULL; +ALTER TABLE `t_ds_fav_task` MODIFY COLUMN `id` bigint NOT NULL AUTO_INCREMENT COMMENT 'id'; +ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `cache_key` varchar(200) NULL COMMENT 'cache_key', MODIFY COLUMN `executor_name` varchar(64) NULL; +SET FOREIGN_KEY_CHECKS = 1; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql index 7a30f2172f..70bdfaa23b 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -14,97 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ --- uc_dolphin_T_t_ds_command_R_test_flag -delimiter ; -DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_command_R_test_flag(); -delimiter d// -CREATE FUNCTION uc_dolphin_T_t_ds_command_R_test_flag() RETURNS void AS $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_command' - AND COLUMN_NAME ='test_flag') - THEN -ALTER TABLE t_ds_command alter column test_flag type int; -ALTER TABLE t_ds_command alter column test_flag set DEFAULT NULL; -END IF; -END; -$$ LANGUAGE plpgsql; -d// -delimiter ; -select uc_dolphin_T_t_ds_command_R_test_flag(); -DROP FUNCTION uc_dolphin_T_t_ds_command_R_test_flag(); --- uc_dolphin_T_t_ds_error_command_R_test_flag -delimiter ; -DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_error_command_R_test_flag(); -delimiter d// -CREATE FUNCTION uc_dolphin_T_t_ds_error_command_R_test_flag() RETURNS void AS $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_error_command' - AND COLUMN_NAME ='test_flag') - THEN -ALTER TABLE t_ds_error_command alter column test_flag type int; -ALTER TABLE t_ds_error_command alter column test_flag set DEFAULT NULL; -END IF; -END; -$$ LANGUAGE plpgsql; -d// -delimiter ; -select uc_dolphin_T_t_ds_error_command_R_test_flag(); -DROP FUNCTION uc_dolphin_T_t_ds_error_command_R_test_flag(); - --- uc_dolphin_T_t_ds_process_instance_R_test_flag -delimiter ; -DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_instance_R_test_flag(); -delimiter d// -CREATE FUNCTION uc_dolphin_T_t_ds_process_instance_R_test_flag() RETURNS void AS $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_process_instance' - AND COLUMN_NAME ='test_flag') - THEN -ALTER TABLE t_ds_process_instance alter column test_flag type int; -ALTER TABLE t_ds_process_instance alter column test_flag set DEFAULT NULL; -END IF; -END; -$$ LANGUAGE plpgsql; -d// -delimiter ; -select uc_dolphin_T_t_ds_process_instance_R_test_flag(); -DROP FUNCTION uc_dolphin_T_t_ds_process_instance_R_test_flag(); - --- uc_dolphin_T_t_ds_task_instance_R_test_flag -delimiter ; -DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_R_test_flag(); -delimiter d// -CREATE FUNCTION uc_dolphin_T_t_ds_task_instance_R_test_flag() RETURNS void AS $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_task_instance' - AND COLUMN_NAME ='test_flag') - THEN -ALTER TABLE t_ds_task_instance alter column test_flag type int; -ALTER TABLE t_ds_task_instance alter column test_flag set DEFAULT NULL; -END IF; -END; -$$ LANGUAGE plpgsql; -d// -delimiter ; -select uc_dolphin_T_t_ds_task_instance_R_test_flag(); -DROP FUNCTION uc_dolphin_T_t_ds_task_instance_R_test_flag(); - -delimiter d// -DROP TABLE IF EXISTS t_ds_trigger_relation; -CREATE TABLE t_ds_trigger_relation ( +-- create table +CREATE TABLE IF NOT EXISTS t_ds_trigger_relation ( id serial NOT NULL, trigger_type int NOT NULL, trigger_code bigint NOT NULL, @@ -114,204 +26,145 @@ CREATE TABLE t_ds_trigger_relation ( PRIMARY KEY (id), CONSTRAINT t_ds_trigger_relation_unique UNIQUE (trigger_type,job_id,trigger_code) ); -d// -delimiter ; +CREATE TABLE IF NOT EXISTS t_ds_relation_sub_workflow ( + id serial NOT NULL, + parent_workflow_instance_id BIGINT NOT NULL, + parent_task_code BIGINT NOT NULL, + sub_workflow_instance_id BIGINT NOT NULL, + PRIMARY KEY (id) +); +CREATE TABLE if not exists "t_ds_fav_task" ( + "id" serial NOT NULL, + "task_type" VARCHAR(64) NOT NULL, + "user_id" integer NOT NULL, + PRIMARY KEY ("id") +); +CREATE TABLE if not exists "t_ds_project_preference" ( + "id" int NOT NULL, + "code" bigint NOT NULL, + "project_code" bigint NOT NULL, + "preferences" VARCHAR(512) NOT NULL, + "user_id" integer NULL, + "state" integer NULL DEFAULT 1, + "create_time" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "update_time" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id") +); +CREATE TABLE if not exists "t_ds_project_parameter" ( + "id" int NOT NULL, + "param_name" VARCHAR(255) NOT NULL, + "param_value" VARCHAR(255) NOT NULL, + "code" bigint NOT NULL, + "project_code" bigint NOT NULL, + "user_id" integer NULL, + "create_time" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "update_time" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id") +); -ALTER TABLE t_ds_task_definition DROP COLUMN IF EXISTS is_cache; +-- add column, if you want to add constraint to the new column you should add them first ALTER TABLE t_ds_task_definition ADD COLUMN IF NOT EXISTS is_cache int DEFAULT '0'; - -ALTER TABLE t_ds_task_definition_log DROP COLUMN IF EXISTS is_cache; ALTER TABLE t_ds_task_definition_log ADD COLUMN IF NOT EXISTS is_cache int DEFAULT '0'; - -ALTER TABLE t_ds_task_instance DROP COLUMN IF EXISTS is_cache; ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS is_cache int DEFAULT '0'; - ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS cache_key varchar(200) DEFAULT NULL; -ALTER TABLE t_ds_task_instance DROP COLUMN IF EXISTS cacke_key; +ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS process_instance_name varchar(255); +ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS executor_name varchar(64); +ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS test_flag int; +ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS project_code int; +ALTER TABLE t_ds_process_instance ADD COLUMN IF NOT EXISTS project_code bigint; +ALTER TABLE t_ds_process_instance ADD COLUMN IF NOT EXISTS executor_name varchar(64); +ALTER TABLE t_ds_process_instance ADD COLUMN IF NOT EXISTS tenant_code varchar(64); +ALTER TABLE t_ds_process_instance ADD COLUMN IF NOT EXISTS project_code bigint; +ALTER TABLE t_ds_process_instance ADD COLUMN IF NOT EXISTS test_flag int; +ALTER TABLE t_ds_command ADD COLUMN IF NOT EXISTS tenant_code varchar(64); +ALTER TABLE t_ds_command ADD COLUMN IF NOT EXISTS test_flag int; +ALTER TABLE t_ds_error_command ADD COLUMN IF NOT EXISTS tenant_code varchar(64); +ALTER TABLE t_ds_error_command ADD COLUMN IF NOT EXISTS test_flag int; +ALTER TABLE t_ds_schedules ADD COLUMN IF NOT EXISTS tenant_code varchar(64); +ALTER TABLE t_ds_alert ADD COLUMN IF NOT EXISTS title varchar(512); +ALTER TABLE t_ds_command ADD COLUMN IF NOT EXISTS worker_group varchar(255); +ALTER TABLE t_ds_project ADD COLUMN IF NOT EXISTS name varchar(255); +ALTER TABLE t_ds_schedules ADD COLUMN IF NOT EXISTS worker_group varchar(255); +ALTER TABLE t_ds_task_instance ADD COLUMN IF NOT EXISTS worker_group varchar(255); +ALTER TABLE t_ds_udfs ADD COLUMN IF NOT EXISTS func_name varchar(255); +ALTER TABLE t_ds_version ADD COLUMN IF NOT EXISTS version varchar(63); +ALTER TABLE t_ds_plugin_define ADD COLUMN IF NOT EXISTS plugin_name varchar(255); +ALTER TABLE t_ds_plugin_define ADD COLUMN IF NOT EXISTS plugin_type varchar(63); +ALTER TABLE t_ds_alert_plugin_instance ADD COLUMN IF NOT EXISTS instance_name varchar(255); +ALTER TABLE t_ds_dq_rule ADD COLUMN IF NOT EXISTS name varchar(255); +ALTER TABLE t_ds_environment ADD COLUMN IF NOT EXISTS name varchar(255); +ALTER TABLE t_ds_task_group_queue ADD COLUMN IF NOT EXISTS task_name VARCHAR(255); +ALTER TABLE t_ds_task_group ADD COLUMN IF NOT EXISTS name varchar(255); +ALTER TABLE t_ds_k8s ADD COLUMN IF NOT EXISTS k8s_name VARCHAR(255); +ALTER TABLE t_ds_k8s_namespace ADD COLUMN IF NOT EXISTS namespace varchar(255); +ALTER TABLE t_ds_cluster ADD COLUMN IF NOT EXISTS name varchar(255); +ALTER TABLE "t_ds_fav_task" ADD COLUMN IF NOT EXISTS "task_type" varchar(64) NOT NULL; + +-- alter column +ALTER TABLE "t_ds_alert" ALTER COLUMN "title" TYPE VARCHAR(512); +ALTER TABLE "t_ds_alert_plugin_instance" ALTER COLUMN "instance_name" TYPE VARCHAR(255); +ALTER TABLE "t_ds_cluster" ALTER COLUMN "name" TYPE VARCHAR(255); +ALTER TABLE "t_ds_command" ALTER COLUMN "worker_group" TYPE VARCHAR(255), ALTER COLUMN "tenant_code" SET DEFAULT 'default'; +ALTER TABLE "t_ds_dq_rule" ALTER COLUMN "name" TYPE VARCHAR(255); +ALTER TABLE "t_ds_environment" ALTER COLUMN "name" TYPE VARCHAR(255); +ALTER TABLE "t_ds_error_command" ALTER COLUMN "worker_group" TYPE VARCHAR(255), ALTER COLUMN "tenant_code" SET DEFAULT 'default'; +ALTER TABLE "t_ds_k8s" ALTER COLUMN "k8s_name" TYPE VARCHAR(255); +ALTER TABLE "t_ds_k8s_namespace" ALTER COLUMN "namespace" TYPE VARCHAR(255), ALTER COLUMN "code" DROP DEFAULT, ALTER COLUMN "cluster_code" DROP DEFAULT; +ALTER TABLE "t_ds_plugin_define" ALTER COLUMN "plugin_name" TYPE VARCHAR(255), ALTER COLUMN "plugin_type" TYPE VARCHAR(63); +ALTER TABLE "t_ds_process_instance" ALTER COLUMN "worker_group" TYPE VARCHAR(255), ALTER COLUMN "executor_name" SET DEFAULT NULL::VARCHAR, ALTER COLUMN "tenant_code" SET DEFAULT 'default'; +ALTER TABLE "t_ds_project" ALTER COLUMN "name" TYPE VARCHAR(255); +ALTER TABLE "t_ds_schedules" ALTER COLUMN "worker_group" TYPE VARCHAR(255), ALTER COLUMN "tenant_code" SET DEFAULT 'default'; +ALTER TABLE "t_ds_task_group" ALTER COLUMN "name" TYPE VARCHAR(255); +ALTER TABLE "t_ds_task_group_queue" ALTER COLUMN "task_name" TYPE VARCHAR(255); +ALTER TABLE "t_ds_task_instance" ALTER COLUMN "worker_group" TYPE VARCHAR(255), ALTER COLUMN "process_instance_name" SET DEFAULT NULL::VARCHAR, ALTER COLUMN "executor_name" SET DEFAULT NULL::VARCHAR, ALTER COLUMN "project_code" TYPE bigint; +ALTER TABLE "t_ds_udfs" ALTER COLUMN "func_name" TYPE VARCHAR(255); +ALTER TABLE "t_ds_user" ALTER COLUMN "tenant_id" SET DEFAULT -1; +ALTER TABLE "t_ds_version" ALTER COLUMN "version" TYPE VARCHAR(63); +ALTER TABLE "t_ds_worker_group" ALTER COLUMN "description" TYPE text, ALTER COLUMN "description" DROP DEFAULT; +ALTER TABLE "t_ds_error_command" ALTER COLUMN "id" DROP DEFAULT; +ALTER TABLE "t_ds_task_instance" ALTER COLUMN "log_path" DROP DEFAULT; +-- create index +DROP INDEX IF EXISTS idx_code; +DROP INDEX IF EXISTS idx_process_code_version; +DROP INDEX IF EXISTS idx_code_version; +-- re index table t_ds_task_definition with index name task_definition_index +DROP INDEX IF EXISTS task_definition_index; CREATE INDEX IF NOT EXISTS idx_cache_key ON t_ds_task_instance USING Btree("cache_key"); +CREATE INDEX IF NOT EXISTS idx_parent_workflow_instance_id ON t_ds_relation_sub_workflow (parent_workflow_instance_id); +CREATE INDEX IF NOT EXISTS idx_parent_task_code ON t_ds_relation_sub_workflow (parent_task_code); +CREATE INDEX IF NOT EXISTS idx_sub_workflow_instance_id ON t_ds_relation_sub_workflow (sub_workflow_instance_id); +CREATE INDEX IF NOT EXISTS task_definition_index ON t_ds_task_definition (project_code, id); +CREATE UNIQUE INDEX IF NOT EXISTS "unique_project_parameter_code" ON "t_ds_project_parameter" ("code"); +CREATE UNIQUE INDEX IF NOT EXISTS "unique_project_parameter_name" ON "t_ds_project_parameter" ("project_code", "param_name"); +CREATE UNIQUE INDEX IF NOT EXISTS "unique_project_preference_code" ON "t_ds_project_preference" ("code"); +CREATE UNIQUE INDEX IF NOT EXISTS "unique_project_preference_project_code" ON "t_ds_project_preference" ("project_code"); --- add_t_ds_process_instance_add_project_code -delimiter ; -DROP FUNCTION IF EXISTS add_t_ds_process_instance_add_project_code(); -delimiter d// -CREATE FUNCTION add_t_ds_process_instance_add_project_code() RETURNS void AS $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_process_instance' - AND COLUMN_NAME ='project_code') - THEN -ALTER TABLE t_ds_process_instance ADD project_code bigint; -ALTER TABLE t_ds_process_instance alter column project_code set DEFAULT NULL; +-- SEQUENCE +DROP SEQUENCE IF EXISTS t_ds_project_preference_id_sequence; +CREATE SEQUENCE t_ds_project_preference_id_sequence; +ALTER TABLE t_ds_project_preference ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_project_preference_id_sequence'); +DROP SEQUENCE IF EXISTS t_ds_project_parameter_id_sequence; +CREATE SEQUENCE t_ds_project_parameter_id_sequence; +ALTER TABLE t_ds_project_parameter ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_project_parameter_id_sequence'); +DROP SEQUENCE IF EXISTS "t_ds_error_command_id_seq"; + +-- add comment comment on column t_ds_process_instance.project_code is 'project code'; -END IF; -IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_process_instance' - AND COLUMN_NAME ='executor_name') - THEN -ALTER TABLE t_ds_process_instance ADD executor_name varchar(64); -ALTER TABLE t_ds_process_instance alter column executor_name set DEFAULT NULL; comment on column t_ds_process_instance.executor_name is 'execute user name'; -END IF; -IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_process_instance' - AND COLUMN_NAME ='tenant_code') - THEN -ALTER TABLE t_ds_process_instance ADD tenant_code varchar(64); -ALTER TABLE t_ds_process_instance alter column tenant_code set DEFAULT NULL; comment on column t_ds_process_instance.tenant_code is 'tenant code'; -END IF; -END; -$$ LANGUAGE plpgsql; -d// -delimiter ; -select add_t_ds_process_instance_add_project_code(); -DROP FUNCTION add_t_ds_process_instance_add_project_code(); - --- add_t_ds_process_instance_add_project_code -delimiter ; -DROP FUNCTION IF EXISTS add_t_ds_task_instance_add_project_code(); -delimiter d// -CREATE FUNCTION add_t_ds_task_instance_add_project_code() RETURNS void AS $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_task_instance' - AND COLUMN_NAME ='process_instance_name') - THEN -ALTER TABLE t_ds_task_instance ADD process_instance_name varchar(255); -ALTER TABLE t_ds_task_instance alter column process_instance_name set DEFAULT NULL; comment on column t_ds_task_instance.process_instance_name is 'process instance name'; -END IF; -IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_task_instance' - AND COLUMN_NAME ='project_code') - THEN -ALTER TABLE t_ds_process_instance ADD project_code bigint; -ALTER TABLE t_ds_process_instance alter column project_code set DEFAULT NULL; comment on column t_ds_process_instance.project_code is 'project code'; -END IF; -IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_task_instance' - AND COLUMN_NAME ='executor_name') - THEN -ALTER TABLE t_ds_task_instance ADD executor_name varchar(64); -ALTER TABLE t_ds_task_instance alter column executor_name set DEFAULT NULL; comment on column t_ds_task_instance.executor_name is 'execute user name'; -END IF; -END; -$$ LANGUAGE plpgsql; -d// -delimiter ; -select add_t_ds_task_instance_add_project_code(); -DROP FUNCTION add_t_ds_task_instance_add_project_code(); - -ALTER TABLE t_ds_alert alter column title type varchar(512); -ALTER TABLE t_ds_alert alter column title set DEFAULT NULL; -ALTER TABLE t_ds_command alter column worker_group type varchar(255); -ALTER TABLE t_ds_project alter column name type varchar(255); -ALTER TABLE t_ds_project alter column name set DEFAULT NULL; -ALTER TABLE t_ds_schedules alter column worker_group type varchar(255); -ALTER TABLE t_ds_task_instance alter column worker_group type varchar(255); -ALTER TABLE t_ds_udfs alter column func_name type varchar(255); -ALTER TABLE t_ds_udfs alter column func_name set NOT NULL ; -ALTER TABLE t_ds_version alter column version type varchar(63); -ALTER TABLE t_ds_version alter column version set NOT NULL; -ALTER TABLE t_ds_plugin_define alter column plugin_name type varchar(255); -ALTER TABLE t_ds_plugin_define alter column plugin_name set NOT NULL; -ALTER TABLE t_ds_plugin_define alter column plugin_type type varchar(63); -ALTER TABLE t_ds_plugin_define alter column plugin_type set NOT NULL; -ALTER TABLE t_ds_alert_plugin_instance alter column instance_name type varchar(255); -ALTER TABLE t_ds_alert_plugin_instance alter column instance_name set DEFAULT NULL; -ALTER TABLE t_ds_dq_rule alter column name type varchar(255); -ALTER TABLE t_ds_dq_rule alter column name set DEFAULT NULL; -ALTER TABLE t_ds_environment alter column name type varchar(255); -ALTER TABLE t_ds_environment alter column name set DEFAULT NULL; -ALTER TABLE t_ds_task_group_queue alter column task_name type VARCHAR(255); -ALTER TABLE t_ds_task_group_queue alter column task_name set DEFAULT NULL ; -ALTER TABLE t_ds_task_group alter column name type varchar(255); -ALTER TABLE t_ds_task_group alter column name set DEFAULT NULL ; -ALTER TABLE t_ds_k8s alter column k8s_name type VARCHAR(255); -ALTER TABLE t_ds_k8s alter column k8s_name set DEFAULT NULL ; -ALTER TABLE t_ds_k8s_namespace alter column namespace type varchar(255); -ALTER TABLE t_ds_k8s_namespace alter column namespace set DEFAULT NULL; -ALTER TABLE t_ds_cluster alter column name type varchar(255); -ALTER TABLE t_ds_cluster alter column name set DEFAULT NULL; - --- tenant improvement -delimiter ; -DROP FUNCTION IF EXISTS add_improvement_workflow_run_tenant(); -delimiter d// -CREATE FUNCTION add_improvement_workflow_run_tenant() RETURNS void AS $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_command' - AND COLUMN_NAME ='tenant_code') - THEN -ALTER TABLE t_ds_command ADD tenant_code varchar(64); -ALTER TABLE t_ds_command alter column tenant_code set DEFAULT 'default'; comment on column t_ds_command.tenant_code is 'tenant code'; -END IF; -IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_error_command' - AND COLUMN_NAME ='tenant_code') - THEN -ALTER TABLE t_ds_error_command ADD tenant_code varchar(64); -ALTER TABLE t_ds_error_command alter column tenant_code set DEFAULT 'default'; comment on column t_ds_error_command.tenant_code is 'tenant code'; -END IF; -IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS - WHERE TABLE_CATALOG=current_database() - AND TABLE_SCHEMA=current_schema() - AND TABLE_NAME='t_ds_schedules' - AND COLUMN_NAME ='tenant_code') - THEN -ALTER TABLE t_ds_schedules ADD tenant_code varchar(64); -ALTER TABLE t_ds_schedules alter column tenant_code set DEFAULT NULL; comment on column t_ds_schedules.tenant_code is 'tenant code'; -END IF; -END; -$$ LANGUAGE plpgsql; -d// -delimiter ; -select add_improvement_workflow_run_tenant(); -DROP FUNCTION add_improvement_workflow_run_tenant(); - --- uc_dolphin_T_t_ds_relation_sub_workflow -CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_relation_sub_workflow() -RETURNS VOID AS $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name='t_ds_relation_sub_workflow' - AND table_schema=current_schema() - ) THEN -CREATE TABLE t_ds_relation_sub_workflow ( - id serial NOT NULL, - parent_workflow_instance_id BIGINT NOT NULL, - parent_task_code BIGINT NOT NULL, - sub_workflow_instance_id BIGINT NOT NULL, - PRIMARY KEY (id) -); -CREATE INDEX idx_parent_workflow_instance_id ON t_ds_relation_sub_workflow (parent_workflow_instance_id); -CREATE INDEX idx_parent_task_code ON t_ds_relation_sub_workflow (parent_task_code); -CREATE INDEX idx_sub_workflow_instance_id ON t_ds_relation_sub_workflow (sub_workflow_instance_id); -END IF; -END; -$$ LANGUAGE plpgsql; +COMMENT ON COLUMN "t_ds_command" ."tenant_code" IS ''; +COMMENT ON COLUMN "t_ds_error_command" ."tenant_code" IS ''; +COMMENT ON COLUMN "t_ds_process_instance" ."project_code" IS ''; +COMMENT ON COLUMN "t_ds_process_instance" ."executor_name" IS ''; +COMMENT ON COLUMN "t_ds_process_instance" ."tenant_code" IS ''; +COMMENT ON COLUMN "t_ds_schedules" ."tenant_code" IS ''; +COMMENT ON COLUMN "t_ds_task_instance" ."process_instance_name" IS ''; +COMMENT ON COLUMN "t_ds_task_instance" ."executor_name" IS ''; +COMMENT ON COLUMN "t_ds_alert" ."sign" IS 'sign=sha1(content)'; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl_post.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl_post.sql index 3a77148a20..887e227afc 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl_post.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl_post.sql @@ -15,6 +15,7 @@ * limitations under the License. */ -ALTER TABLE "t_ds_process_definition" DROP "tenant_id"; -ALTER TABLE "t_ds_process_definition_log" DROP "tenant_id"; -ALTER TABLE "t_ds_process_instance" DROP "tenant_id"; +ALTER TABLE "t_ds_process_definition" DROP COLUMN IF EXISTS "tenant_id"; +ALTER TABLE "t_ds_process_definition_log" DROP COLUMN IF EXISTS "tenant_id"; +ALTER TABLE "t_ds_process_instance" DROP COLUMN IF EXISTS "tenant_id"; +ALTER TABLE "t_ds_fav_task" DROP COLUMN IF EXISTS "task_name"; \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_dml.sql index 33b7b16c20..482b317a60 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_dml.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_dml.sql @@ -18,7 +18,7 @@ INSERT INTO t_ds_tenant(id, tenant_code, description, queue_id, create_time, update_time) VALUES (-1, 'default', 'default tenant', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22') ON CONFLICT (id) DO NOTHING; -- tenant improvement -UPDATE t_ds_schedules t1 SET t1.tenant_code = COALESCE(t3.tenant_code, 'default') FROM t_ds_process_definition t2 LEFT JOIN t_ds_tenant t3 ON t2.tenant_id = t3.id WHERE t1.process_definition_code = t2.code; +UPDATE t_ds_schedules as t1 SET tenant_code = COALESCE(t3.tenant_code, 'default') FROM t_ds_process_definition as t2 LEFT JOIN t_ds_tenant t3 ON t2.tenant_id = t3.id WHERE t1.process_definition_code = t2.code; UPDATE t_ds_process_instance SET tenant_code = 'default' WHERE tenant_code IS NULL; -- data quality support choose database diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl_post.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl_post.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl_post.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl_post.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl_post.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl_post.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java index 2ad7940905..ba7e5d28e9 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java @@ -27,10 +27,12 @@ import javax.sql.DataSource; import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; @Service @Slf4j +@Profile("mysql") public class MySQLUpgradeDao extends UpgradeDao { private MySQLUpgradeDao(DataSource dataSource) { diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java index 352da3e9d6..c2d4b637f1 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java @@ -28,10 +28,12 @@ import javax.sql.DataSource; import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; @Service @Slf4j +@Profile("postgresql") public class PostgreSQLUpgradeDao extends UpgradeDao { private PostgreSQLUpgradeDao(DataSource dataSource) {