From fba47b337e3befec2fc104f9de30bda6986d7f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E6=99=93?= Date: Mon, 24 Apr 2023 12:08:10 +0800 Subject: [PATCH] [BUG][Data Quality] Fixing the incorrect DML statements in upgrade SQL. (#13960) --- .../postgresql/dolphinscheduler_dml.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/postgresql/dolphinscheduler_dml.sql index 8c84a54d7a..a3ab5ed98d 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/postgresql/dolphinscheduler_dml.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/postgresql/dolphinscheduler_dml.sql @@ -20,8 +20,8 @@ INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "nam INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(4, 'MonthlyAvg', 'select round(avg(statistics_value),2) as month_avg from t_ds_dq_task_statistics_value where data_time >= date_trunc(''MONTH'', ${data_time}) and data_time = date_add(date_trunc(''day'', ${data_time}),-7) and data_time = date_add(date_trunc(''day'', ${data_time}),-30) and data_time < date_trunc(''day'', ${data_time}) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'last_thirty_days', 'last_thirty_days.last_30_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true); -INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(7, 'SrcTableTotalRows', 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})@', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); -INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(8, 'TargetTableTotalRows', 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})@', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); +INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(7, 'SrcTableTotalRows', 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); +INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(8, 'TargetTableTotalRows', 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time) VALUES(1, '$t(null_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time) VALUES(2, '$t(custom_sql)', 1, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); @@ -35,10 +35,10 @@ INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time) INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time) VALUES(10, '$t(table_count_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(1, 1, 'SELECT COUNT(*) AS nulls FROM null_items', 'null_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(2, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})@', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); +INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(2, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(3, 1, 'SELECT COUNT(*) AS miss from miss_items', 'miss_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(4, 1, 'SELECT COUNT(*) AS valids FROM invalid_length_items', 'invalid_length_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(5, 1, 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})@', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); +INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(5, 1, 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(6, 1, 'SELECT ${src_field} FROM ${src_table} group by ${src_field} having count(*) > 1', 'duplicate_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(7, 1, 'SELECT COUNT(*) AS duplicates FROM duplicate_items', 'duplicate_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(8, 1, 'SELECT ${src_table}.* FROM (SELECT * FROM ${src_table} WHERE (${src_filter})) ${src_table} LEFT JOIN (SELECT * FROM ${target_table} WHERE (${target_filter})) ${target_table} ON ${on_clause} WHERE ${where_clause}', 'miss_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); @@ -48,9 +48,9 @@ INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", i INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(12, 1, 'SELECT COUNT(*) AS timeliness FROM timeliness_items', 'timeliness_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(13, 1, 'SELECT * FROM ${src_table} where (${src_field} not in ( ${enum_list} ) or ${src_field} is null) AND (${src_filter}) ', 'enum_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(14, 1, 'SELECT COUNT(*) AS enums FROM enum_items', 'enum_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(15, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})@', 'table_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(16, 1, 'SELECT * FROM ${src_table} WHERE (${src_field} is null or ${src_field} = '''') AND (${src_filter})@', 'null_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(17, 1, 'SELECT * FROM ${src_table} WHERE (length(${src_field}) ${logic_operator} ${field_length}) AND (${src_filter})@', 'invalid_length_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); +INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(15, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'table_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); +INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(16, 1, 'SELECT * FROM ${src_table} WHERE (${src_field} is null or ${src_field} = '''') AND (${src_filter})', 'null_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); +INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(17, 1, 'SELECT * FROM ${src_table} WHERE (length(${src_field}) ${logic_operator} ${field_length}) AND (${src_filter})', 'invalid_length_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_input_entry (id, field, "type", title, "value", "options", placeholder, option_source_type, value_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) VALUES(1, 'src_connector_type', 'select', '$t(src_connector_type)', '', '[{"label":"HIVE","value":"HIVE"},{"label":"JDBC","value":"JDBC"}]', 'please select source connector type', 2, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_input_entry (id, field, "type", title, "value", "options", placeholder, option_source_type, value_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) VALUES(2, 'src_datasource_id', 'select', '$t(src_datasource_id)', '', NULL, 'please select source datasource id', 1, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');