github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[BUG][Data Quality] Fixing the incorrect DML statements in upgrade SQL. (#13960)

3.2.0-release
破晓 2 years ago committed by GitHub
parent
73b505f639
commit
fba47b337e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 7 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 14
      dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/postgresql/dolphinscheduler_dml.sql

14
dolphinscheduler-dao/src/main/resources/sql/upgrade/3.0.0_schema/postgresql/dolphinscheduler_dml.sql
Unescape View File

@ -20,8 +20,8 @@ INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "nam
INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(4, 'MonthlyAvg', 'select round(avg(statistics_value),2) as month_avg from t_ds_dq_task_statistics_value where data_time >= date_trunc(''MONTH'', ${data_time}) and data_time <date_trunc(''day'', ${data_time}) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'month_range', 'month_range.month_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true); INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(4, 'MonthlyAvg', 'select round(avg(statistics_value),2) as month_avg from t_ds_dq_task_statistics_value where data_time >= date_trunc(''MONTH'', ${data_time}) and data_time <date_trunc(''day'', ${data_time}) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'month_range', 'month_range.month_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true);
INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(5, 'Last7DayAvg', 'select round(avg(statistics_value),2) as last_7_avg from t_ds_dq_task_statistics_value where data_time >= date_add(date_trunc(''day'', ${data_time}),-7) and data_time <date_trunc(''day'', ${data_time}) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'last_seven_days', 'last_seven_days.last_7_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true); INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(5, 'Last7DayAvg', 'select round(avg(statistics_value),2) as last_7_avg from t_ds_dq_task_statistics_value where data_time >= date_add(date_trunc(''day'', ${data_time}),-7) and data_time <date_trunc(''day'', ${data_time}) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'last_seven_days', 'last_seven_days.last_7_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true);
INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(6, 'Last30DayAvg', 'select round(avg(statistics_value),2) as last_30_avg from t_ds_dq_task_statistics_value where data_time >= date_add(date_trunc(''day'', ${data_time}),-30) and data_time < date_trunc(''day'', ${data_time}) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'last_thirty_days', 'last_thirty_days.last_30_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true); INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(6, 'Last30DayAvg', 'select round(avg(statistics_value),2) as last_30_avg from t_ds_dq_task_statistics_value where data_time >= date_add(date_trunc(''day'', ${data_time}),-30) and data_time < date_trunc(''day'', ${data_time}) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'last_thirty_days', 'last_thirty_days.last_30_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true);
INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(7, 'SrcTableTotalRows', 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})@', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(7, 'SrcTableTotalRows', 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false);
INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(8, 'TargetTableTotalRows', 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})@', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); INSERT INTO t_ds_dq_comparison_type (id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) VALUES(8, 'TargetTableTotalRows', 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false);
INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time) VALUES(1, '$t(null_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time) VALUES(1, '$t(null_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000');
INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time) VALUES(2, '$t(custom_sql)', 1, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time) VALUES(2, '$t(custom_sql)', 1, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000');
@ -35,10 +35,10 @@ INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time)
INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time) VALUES(10, '$t(table_count_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); INSERT INTO t_ds_dq_rule (id, "name", "type", user_id, create_time, update_time) VALUES(10, '$t(table_count_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(1, 1, 'SELECT COUNT(*) AS nulls FROM null_items', 'null_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(1, 1, 'SELECT COUNT(*) AS nulls FROM null_items', 'null_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(2, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})@', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(2, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(3, 1, 'SELECT COUNT(*) AS miss from miss_items', 'miss_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(3, 1, 'SELECT COUNT(*) AS miss from miss_items', 'miss_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(4, 1, 'SELECT COUNT(*) AS valids FROM invalid_length_items', 'invalid_length_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(4, 1, 'SELECT COUNT(*) AS valids FROM invalid_length_items', 'invalid_length_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(5, 1, 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})@', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(5, 1, 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(6, 1, 'SELECT ${src_field} FROM ${src_table} group by ${src_field} having count(*) > 1', 'duplicate_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(6, 1, 'SELECT ${src_field} FROM ${src_table} group by ${src_field} having count(*) > 1', 'duplicate_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(7, 1, 'SELECT COUNT(*) AS duplicates FROM duplicate_items', 'duplicate_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(7, 1, 'SELECT COUNT(*) AS duplicates FROM duplicate_items', 'duplicate_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(8, 1, 'SELECT ${src_table}.* FROM (SELECT * FROM ${src_table} WHERE (${src_filter})) ${src_table} LEFT JOIN (SELECT * FROM ${target_table} WHERE (${target_filter})) ${target_table} ON ${on_clause} WHERE ${where_clause}', 'miss_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(8, 1, 'SELECT ${src_table}.* FROM (SELECT * FROM ${src_table} WHERE (${src_filter})) ${src_table} LEFT JOIN (SELECT * FROM ${target_table} WHERE (${target_filter})) ${target_table} ON ${on_clause} WHERE ${where_clause}', 'miss_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
@ -48,9 +48,9 @@ INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", i
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(12, 1, 'SELECT COUNT(*) AS timeliness FROM timeliness_items', 'timeliness_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(12, 1, 'SELECT COUNT(*) AS timeliness FROM timeliness_items', 'timeliness_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(13, 1, 'SELECT * FROM ${src_table} where (${src_field} not in ( ${enum_list} ) or ${src_field} is null) AND (${src_filter}) ', 'enum_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(13, 1, 'SELECT * FROM ${src_table} where (${src_field} not in ( ${enum_list} ) or ${src_field} is null) AND (${src_filter}) ', 'enum_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(14, 1, 'SELECT COUNT(*) AS enums FROM enum_items', 'enum_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(14, 1, 'SELECT COUNT(*) AS enums FROM enum_items', 'enum_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(15, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})@', 'table_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(15, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'table_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(16, 1, 'SELECT * FROM ${src_table} WHERE (${src_field} is null or ${src_field} = '''') AND (${src_filter})@', 'null_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(16, 1, 'SELECT * FROM ${src_table} WHERE (${src_field} is null or ${src_field} = '''') AND (${src_filter})', 'null_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(17, 1, 'SELECT * FROM ${src_table} WHERE (length(${src_field}) ${logic_operator} ${field_length}) AND (${src_filter})@', 'invalid_length_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_execute_sql (id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) VALUES(17, 1, 'SELECT * FROM ${src_table} WHERE (length(${src_field}) ${logic_operator} ${field_length}) AND (${src_filter})', 'invalid_length_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_input_entry (id, field, "type", title, "value", "options", placeholder, option_source_type, value_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) VALUES(1, 'src_connector_type', 'select', '$t(src_connector_type)', '', '[{"label":"HIVE","value":"HIVE"},{"label":"JDBC","value":"JDBC"}]', 'please select source connector type', 2, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_input_entry (id, field, "type", title, "value", "options", placeholder, option_source_type, value_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) VALUES(1, 'src_connector_type', 'select', '$t(src_connector_type)', '', '[{"label":"HIVE","value":"HIVE"},{"label":"JDBC","value":"JDBC"}]', 'please select source connector type', 2, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_dq_rule_input_entry (id, field, "type", title, "value", "options", placeholder, option_source_type, value_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) VALUES(2, 'src_datasource_id', 'select', '$t(src_datasource_id)', '', NULL, 'please select source datasource id', 1, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); INSERT INTO t_ds_dq_rule_input_entry (id, field, "type", title, "value", "options", placeholder, option_source_type, value_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) VALUES(2, 'src_datasource_id', 'select', '$t(src_datasource_id)', '', NULL, 'please select source datasource id', 1, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');

Write Preview
Loading…
Cancel
Save