From 98a8b5383edf695811b64f6d871e8783e4a60003 Mon Sep 17 00:00:00 2001 From: Ivan Date: Tue, 11 Oct 2022 10:20:05 +0800 Subject: [PATCH] [ fix: #11590 ][upgrade] Fix the bugs when I upgrade ds from v1.3.9 to v3.0.0 ,such as file cannot be found or column not found etc (#11619) * fix the bugs which upgrade ds metabase from v1.3.9 to v3.0.0 with mysql * [Fix-11590][upgrade] Fix the bugs when I upgrade ds from v1.3.9 to v3.0.0 ,such as file cannot be found or column not found etc * fix: #11590 add process_instance_code_state_index with table t_ds_process_instance to improve query efficiency for the method of findLastProcessInterval * Fix the bug that cannot find the files when upgrade ds version with postgresql Co-authored-by: fan.wang --- .../mysql/dolphinscheduler_ddl.sql | 6 ++++- .../mysql/dolphinscheduler_ddl_post.sql | 26 ++++++++++++++++++ .../mysql/dolphinscheduler_dml.sql | 16 +++++++++++ .../postgresql/dolphinscheduler_ddl_post.sql | 27 +++++++++++++++++++ .../postgresql/dolphinscheduler_dml.sql | 16 +++++++++++ .../mysql/dolphinscheduler_dml.sql | 16 +++++++++++ .../postgresql/dolphinscheduler_dml.sql | 16 +++++++++++ .../mysql/dolphinscheduler_ddl.sql | 2 +- 8 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl_post.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl_post.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/mysql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/postgresql/dolphinscheduler_dml.sql diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql index 8e45884395..65f4dd1fde 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql @@ -242,6 +242,7 @@ CREATE TABLE `t_ds_process_definition_log` ( `warning_group_id` int(11) DEFAULT NULL COMMENT 'alert group id', `timeout` int(11) DEFAULT '0' COMMENT 'time out,unit: minute', `tenant_id` int(11) NOT NULL DEFAULT '-1' COMMENT 'tenant id', + `execution_type` tinyint(4) DEFAULT '0' COMMENT 'execution_type 0:parallel,1:serial wait,2:serial discard,3:serial priority', `operator` int(11) DEFAULT NULL COMMENT 'operator user id', `operate_time` datetime DEFAULT NULL COMMENT 'operate time', `create_time` datetime NOT NULL COMMENT 'create time', @@ -403,8 +404,10 @@ alter table t_ds_process_instance add process_definition_version int(11) DEFAULT alter table t_ds_process_instance add environment_code bigint(20) DEFAULT '-1' COMMENT 'environment code' AFTER worker_group; alter table t_ds_process_instance add var_pool longtext COMMENT 'var_pool' AFTER tenant_id; alter table t_ds_process_instance add dry_run tinyint(4) DEFAULT '0' COMMENT 'dry run flag：0 normal, 1 dry run' AFTER var_pool; +alter table t_ds_process_instance add next_process_instance_id int(11) DEFAULT '0' COMMENT 'serial queue next processInstanceId' AFTER dry_run; alter table t_ds_process_instance drop KEY `process_instance_index`; alter table t_ds_process_instance add KEY `process_instance_index` (`process_definition_code`,`id`) USING BTREE; +alter table t_ds_process_instance add KEY `process_instance_code_state_index` (`process_definition_code`,`state`) USING BTREE; alter table t_ds_process_instance drop process_instance_json; alter table t_ds_process_instance drop locations; alter table t_ds_process_instance drop connects; @@ -438,4 +441,5 @@ alter table t_ds_process_definition add `warning_group_id` int(11) DEFAULT NULL alter table t_ds_process_definition add UNIQUE KEY `process_unique` (`name`,`project_code`) USING BTREE; alter table t_ds_process_definition modify `description` text COMMENT 'description' after `version`; alter table t_ds_process_definition modify `release_state` tinyint(4) DEFAULT NULL COMMENT 'process definition release state：0:offline,1:online' after `project_code`; -alter table t_ds_process_definition modify `create_time` datetime DEFAULT NULL COMMENT 'create time' after `tenant_id`; +alter table t_ds_process_definition add `execution_type` tinyint(4) DEFAULT '0' COMMENT 'execution_type 0:parallel,1:serial wait,2:serial discard,3:serial priority' after `tenant_id`; +alter table t_ds_process_definition modify `create_time` datetime DEFAULT NULL COMMENT 'create time' after `execution_type`; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl_post.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl_post.sql new file mode 100644 index 0000000000..dfde96209e --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl_post.sql @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +alter table t_ds_process_definition drop primary key, ADD PRIMARY KEY (`id`,`code`); +ALTER TABLE t_ds_process_definition drop KEY `process_definition_unique`; +ALTER TABLE t_ds_process_definition drop KEY `process_definition_index`; +alter table t_ds_process_definition drop process_definition_json; +alter table t_ds_process_definition drop connects; +alter table t_ds_process_definition drop receivers; +alter table t_ds_process_definition drop receivers_cc; +alter table t_ds_process_definition drop modify_by; +alter table t_ds_process_definition drop resource_ids; \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl_post.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl_post.sql new file mode 100644 index 0000000000..728b6bd239 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl_post.sql @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +ALTER TABLE "t_ds_process_definition" DROP CONSTRAINT "t_ds_process_definition_pkey"; +ALTER TABLE "t_ds_process_definition" ADD CONSTRAINT "t_ds_process_definition_pkey" PRIMARY KEY ("id","code"); +ALTER TABLE "t_ds_process_definition" DROP CONSTRAINT "process_definition_unique"; +DROP INDEX "process_definition_index"; +ALTER TABLE "t_ds_process_definition" DROP "process_definition_json"; +ALTER TABLE "t_ds_process_definition" DROP "connects"; +ALTER TABLE "t_ds_process_definition" DROP "receivers"; +ALTER TABLE "t_ds_process_definition" DROP "receivers_cc"; +ALTER TABLE "t_ds_process_definition" DROP "modify_by"; +ALTER TABLE "t_ds_process_definition" DROP "resource_ids"; \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/mysql/dolphinscheduler_ddl.sql index 72acb02051..630984e3b0 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/mysql/dolphinscheduler_ddl.sql @@ -469,7 +469,7 @@ BEGIN ALTER TABLE `t_ds_alert` ADD COLUMN `project_code` bigint DEFAULT NULL COMMENT 'project_code'; ALTER TABLE `t_ds_alert` ADD COLUMN `process_definition_code` bigint DEFAULT NULL COMMENT 'process_definition_code'; ALTER TABLE `t_ds_alert` ADD COLUMN `process_instance_id` int DEFAULT NULL COMMENT 'process_instance_id'; -ALTER TABLE `t_ds_alert` ADD COLUMN `alert_type` int DEFAULT NULL COMMENT 'alert_type'; +ALTER TABLE `t_ds_alert` MODIFY COLUMN `alert_type` int DEFAULT NULL COMMENT 'alert_type'; END IF; END; d//