From 5ea918b96e361194bf0ac922e25863f87bb18a42 Mon Sep 17 00:00:00 2001 From: Wing-Kam Wong Date: Fri, 2 Dec 2022 20:20:22 +0800 Subject: [PATCH 1/4] fix(nocodb): handle comparing a date with empty string in pg --- .../lib/sql/formulav2/formulaQueryBuilderv2.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index 0c382ea3b4..925ab10066 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -55,8 +55,11 @@ export default async function formulaQueryBuilderv2( jsep.plugins.register(jsepCurlyHook); const tree = jsep(_tree); + const columnIdToUidt = {}; + // todo: improve - implement a common solution for filter, sort, formula, etc for (const col of await model.getColumns()) { + columnIdToUidt[col.id] = col.uidt; if (col.id in aliasToColumn) continue; switch (col.uidt) { case UITypes.Formula: @@ -659,6 +662,20 @@ export default async function formulaQueryBuilderv2( const right = fn(pt.right, null, pt.operator).toQuery(); let sql = `${left} ${pt.operator} ${right}${colAlias}`; + // comparing a date with empty string would throw + // `ERROR: zero-length delimited identifier` in Postgres + if ( + knex.clientType() === 'pg' && + columnIdToUidt[pt.left.name] === UITypes.Date && + pt.right.value === '' + ) { + if (pt.operator === '=') { + sql = `${left} IS NULL ${colAlias}`; + } else if (pt.operator === '!=') { + sql = `${left} IS NOT NULL ${colAlias}`; + } + } + // handle NULL values when calling CONCAT for sqlite3 if (pt.left.fnName === 'CONCAT' && knex.clientType() === 'sqlite3') { sql = `COALESCE(${left}, '') ${pt.operator} COALESCE(${right},'')${colAlias}`; From f8b5d2f3fc4c18fdb55e9aadcbc11ad223c7ad20 Mon Sep 17 00:00:00 2001 From: Wing-Kam Wong Date: Sat, 3 Dec 2022 12:31:13 +0800 Subject: [PATCH 2/4] feat(nocodb): add validateDateWithUnknownFormat --- .../lib/sql/helpers/formulaFnHelper.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/formulaFnHelper.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/formulaFnHelper.ts index f8057a2473..e3268d7406 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/formulaFnHelper.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/formulaFnHelper.ts @@ -1,3 +1,7 @@ +import dayjs, { extend } from 'dayjs'; +import customParseFormat from 'dayjs/plugin/customParseFormat.js'; +extend(customParseFormat); + export function getWeekdayByText(v: string) { return { monday: 0, @@ -21,3 +25,28 @@ export function getWeekdayByIndex(idx: number): string { 6: 'sunday', }[idx || 0]; } + +export function validateDateWithUnknownFormat(v: string) { + const dateFormats = [ + 'DD-MM-YYYY', + 'MM-DD-YYYY', + 'YYYY-MM-DD', + 'DD/MM/YYYY', + 'MM/DD/YYYY', + 'YYYY/MM/DD', + 'DD MM YYYY', + 'MM DD YYYY', + 'YYYY MM DD', + ]; + for (const format of dateFormats) { + if (dayjs(v, format, true).isValid() as any) { + return true; + } + for (const timeFormat of ['HH:mm', 'HH:mm:ss', 'HH:mm:ss.SSS']) { + if (dayjs(v, `${format} ${timeFormat}`, true).isValid() as any) { + return true; + } + } + } + return false; +} From d138c79709c90934ac0dfe9d4b73814737eb3482 Mon Sep 17 00:00:00 2001 From: Wing-Kam Wong Date: Sat, 3 Dec 2022 12:31:37 +0800 Subject: [PATCH 3/4] fix(nocodb): handle comparing a date in IF in pg --- .../sql/formulav2/formulaQueryBuilderv2.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index 925ab10066..2478f5a251 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -8,6 +8,7 @@ import { XKnex } from '../../../index'; import LinkToAnotherRecordColumn from '../../../../../models/LinkToAnotherRecordColumn'; import LookupColumn from '../../../../../models/LookupColumn'; import { jsepCurlyHook, UITypes } from 'nocodb-sdk'; +import { validateDateWithUnknownFormat } from '../helpers/formulaFnHelper'; // todo: switch function based on database @@ -666,12 +667,20 @@ export default async function formulaQueryBuilderv2( // `ERROR: zero-length delimited identifier` in Postgres if ( knex.clientType() === 'pg' && - columnIdToUidt[pt.left.name] === UITypes.Date && - pt.right.value === '' + columnIdToUidt[pt.left.name] === UITypes.Date ) { - if (pt.operator === '=') { - sql = `${left} IS NULL ${colAlias}`; - } else if (pt.operator === '!=') { + // The correct way to compare with Date should be using + // `IS_AFTER`, `IS_BEFORE`, or `IS_SAME` + // This is to prevent empty data returned to UI due to incorrect SQL + if (pt.right.value === '') { + if (pt.operator === '=') { + sql = `${left} IS NULL ${colAlias}`; + } else { + sql = `${left} IS NOT NULL ${colAlias}`; + } + } else if (!validateDateWithUnknownFormat(pt.right.value)) { + // left tree value is date but right tree value is not date + // return true if left tree value is not null, else false sql = `${left} IS NOT NULL ${colAlias}`; } } From 402ff5e45e7783524c9504d05b3961b27fad8eee Mon Sep 17 00:00:00 2001 From: Wing-Kam Wong Date: Thu, 8 Dec 2022 19:51:47 +0800 Subject: [PATCH 4/4] fix(nocodb): handle empty left tree for IF case --- .../sql/formulav2/formulaQueryBuilderv2.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index 2478f5a251..d0c60e1a7e 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -684,6 +684,25 @@ export default async function formulaQueryBuilderv2( sql = `${left} IS NOT NULL ${colAlias}`; } } + if ( + knex.clientType() === 'pg' && + columnIdToUidt[pt.right.name] === UITypes.Date + ) { + // The correct way to compare with Date should be using + // `IS_AFTER`, `IS_BEFORE`, or `IS_SAME` + // This is to prevent empty data returned to UI due to incorrect SQL + if (pt.left.value === '') { + if (pt.operator === '=') { + sql = `${right} IS NULL ${colAlias}`; + } else { + sql = `${right} IS NOT NULL ${colAlias}`; + } + } else if (!validateDateWithUnknownFormat(pt.left.value)) { + // right tree value is date but left tree value is not date + // return true if right tree value is not null, else false + sql = `${right} IS NOT NULL ${colAlias}`; + } + } // handle NULL values when calling CONCAT for sqlite3 if (pt.left.fnName === 'CONCAT' && knex.clientType() === 'sqlite3') {