From efb58e8d31f08cd0c7f7e2d2ed724fee51845c56 Mon Sep 17 00:00:00 2001 From: Wing-Kam Wong Date: Thu, 13 Apr 2023 13:57:16 +0800 Subject: [PATCH 1/5] feat(nocodb): add convertDateFormat --- .../lib/sql/helpers/convertDateFormat.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/convertDateFormat.ts diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/convertDateFormat.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/convertDateFormat.ts new file mode 100644 index 0000000000..c9e833e3b3 --- /dev/null +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/convertDateFormat.ts @@ -0,0 +1,23 @@ +export function convertDateFormat(date_format: string, type: string) { + if (date_format === 'YYYY-MM-DD') { + if (type === 'mysql2' || type === 'sqlite') return '%Y-%m-%d'; + } else if (date_format === 'YYYY/MM/DD') { + if (type === 'mysql2' || type === 'sqlite') return '%Y/%m/%d'; + } else if (date_format === 'DD-MM-YYYY') { + if (type === 'mysql2' || type === 'sqlite') return '%d/%m/%Y'; + } else if (date_format === 'MM-DD-YYYY') { + if (type === 'mysql2' || type === 'sqlite') return '%d-%m-%Y'; + } else if (date_format === 'DD/MM/YYYY') { + if (type === 'mysql2' || type === 'sqlite') return '%d/%m/%Y'; + } else if (date_format === 'MM/DD/YYYY') { + if (type === 'mysql2' || type === 'sqlite') return '%m-%d-%Y'; + } else if (date_format === 'DD MM YYYY') { + if (type === 'mysql2' || type === 'sqlite') return '%d %m %Y'; + } else if (date_format === 'MM DD YYYY') { + if (type === 'mysql2' || type === 'sqlite') return '%m %d %Y'; + } else if (date_format === 'YYYY MM DD') { + if (type === 'mysql2' || type === 'sqlite') return '%Y %m %d'; + } + // pg / mssql + return date_format; +} From 64e24b12159ed13128b9a057c92f40c612761b3b Mon Sep 17 00:00:00 2001 From: Wing-Kam Wong Date: Thu, 13 Apr 2023 13:57:48 +0800 Subject: [PATCH 2/5] fix(nocodb): convert date format based on meta in CONCAT --- .../sql/formulav2/formulaQueryBuilderv2.ts | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index f6a93ea247..7a660bf2b0 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -6,8 +6,9 @@ import FormulaColumn from '../../../../../models/FormulaColumn'; import { validateDateWithUnknownFormat } from '../helpers/formulaFnHelper'; import { CacheGetType, CacheScope } from '../../../../../utils/globals'; import NocoCache from '../../../../../cache/NocoCache'; +import Column from '../../../../../models/Column'; +import { convertDateFormat } from '../helpers/convertDateFormat'; import type Model from '../../../../../models/Model'; -import type Column from '../../../../../models/Column'; import type RollupColumn from '../../../../../models/RollupColumn'; import type { XKnex } from '../../../index'; import type LinkToAnotherRecordColumn from '../../../../../models/LinkToAnotherRecordColumn'; @@ -633,8 +634,42 @@ async function _formulaQueryBuilder( `${pt.callee.name}(${( await Promise.all( pt.arguments.map(async (arg) => { - const query = (await fn(arg)).builder.toQuery(); + let query = (await fn(arg)).builder.toQuery(); if (pt.callee.name === 'CONCAT') { + if ( + arg.type === 'Identifier' && + arg.name in columnIdToUidt && + columnIdToUidt[arg.name] === UITypes.Date + ) { + const meta = ( + await Column.get({ + colId: arg.name, + }) + ).meta; + + if (knex.clientType() === 'mysql2') { + query = `DATE_FORMAT(${query}, '${convertDateFormat( + meta.date_format, + knex.clientType() + )}')`; + } else if (knex.clientType() === 'pg') { + query = `TO_CHAR(${query}, '${convertDateFormat( + meta.date_format, + knex.clientType() + )}')`; + } else if (knex.clientType() === 'sqlite') { + query = `strftime('${convertDateFormat( + meta.date_format, + knex.clientType() + )}', ${query})`; + } else if (knex.clientType() === 'mssql') { + query = `FORMAT(${query}, '${convertDateFormat( + meta.date_format, + knex.clientType() + )}')`; + } + } + if (knex.clientType() === 'mysql2') { // mysql2: CONCAT() returns NULL if any argument is NULL. // adding IFNULL to convert NULL values to empty strings From 02fd36589d1d446d9ca5c2ed5cb33802a1fa09ae Mon Sep 17 00:00:00 2001 From: Wing-Kam Wong Date: Thu, 13 Apr 2023 16:30:00 +0800 Subject: [PATCH 3/5] fix(nocodb): sqlite -> sqlite3 --- .../lib/sql/helpers/convertDateFormat.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/convertDateFormat.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/convertDateFormat.ts index c9e833e3b3..97057840f3 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/convertDateFormat.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/convertDateFormat.ts @@ -1,22 +1,22 @@ export function convertDateFormat(date_format: string, type: string) { if (date_format === 'YYYY-MM-DD') { - if (type === 'mysql2' || type === 'sqlite') return '%Y-%m-%d'; + if (type === 'mysql2' || type === 'sqlite3') return '%Y-%m-%d'; } else if (date_format === 'YYYY/MM/DD') { - if (type === 'mysql2' || type === 'sqlite') return '%Y/%m/%d'; + if (type === 'mysql2' || type === 'sqlite3') return '%Y/%m/%d'; } else if (date_format === 'DD-MM-YYYY') { - if (type === 'mysql2' || type === 'sqlite') return '%d/%m/%Y'; + if (type === 'mysql2' || type === 'sqlite3') return '%d/%m/%Y'; } else if (date_format === 'MM-DD-YYYY') { - if (type === 'mysql2' || type === 'sqlite') return '%d-%m-%Y'; + if (type === 'mysql2' || type === 'sqlite3') return '%d-%m-%Y'; } else if (date_format === 'DD/MM/YYYY') { - if (type === 'mysql2' || type === 'sqlite') return '%d/%m/%Y'; + if (type === 'mysql2' || type === 'sqlite3') return '%d/%m/%Y'; } else if (date_format === 'MM/DD/YYYY') { - if (type === 'mysql2' || type === 'sqlite') return '%m-%d-%Y'; + if (type === 'mysql2' || type === 'sqlite3') return '%m-%d-%Y'; } else if (date_format === 'DD MM YYYY') { - if (type === 'mysql2' || type === 'sqlite') return '%d %m %Y'; + if (type === 'mysql2' || type === 'sqlite3') return '%d %m %Y'; } else if (date_format === 'MM DD YYYY') { - if (type === 'mysql2' || type === 'sqlite') return '%m %d %Y'; + if (type === 'mysql2' || type === 'sqlite3') return '%m %d %Y'; } else if (date_format === 'YYYY MM DD') { - if (type === 'mysql2' || type === 'sqlite') return '%Y %m %d'; + if (type === 'mysql2' || type === 'sqlite3') return '%Y %m %d'; } // pg / mssql return date_format; From 841a57cfcf57e882dbdbf31204273895c56e4551 Mon Sep 17 00:00:00 2001 From: Wing-Kam Wong Date: Thu, 13 Apr 2023 16:30:10 +0800 Subject: [PATCH 4/5] feat(nocodb): add convertDateFormatForConcat --- .../lib/sql/helpers/formulaFnHelper.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/formulaFnHelper.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/formulaFnHelper.ts index e3268d7406..5a21ba3261 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/formulaFnHelper.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/formulaFnHelper.ts @@ -1,5 +1,8 @@ import dayjs, { extend } from 'dayjs'; import customParseFormat from 'dayjs/plugin/customParseFormat.js'; +import { UITypes } from 'nocodb-sdk'; +import Column from '../../../../../models/Column'; +import { convertDateFormat } from './convertDateFormat'; extend(customParseFormat); export function getWeekdayByText(v: string) { @@ -50,3 +53,45 @@ export function validateDateWithUnknownFormat(v: string) { } return false; } + +export async function convertDateFormatForConcat( + o, + columnIdToUidt, + query, + clientType +) { + if ( + o?.type === 'Identifier' && + o?.name in columnIdToUidt && + columnIdToUidt[o.name] === UITypes.Date + ) { + const meta = ( + await Column.get({ + colId: o.name, + }) + ).meta; + + if (clientType === 'mysql2') { + query = `DATE_FORMAT(${query}, '${convertDateFormat( + meta.date_format, + clientType + )}')`; + } else if (clientType === 'pg') { + query = `TO_CHAR(${query}, '${convertDateFormat( + meta.date_format, + clientType + )}')`; + } else if (clientType === 'sqlite3') { + query = `strftime('${convertDateFormat( + meta.date_format, + clientType + )}', ${query})`; + } else if (clientType === 'mssql') { + query = `FORMAT(${query}, '${convertDateFormat( + meta.date_format, + clientType + )}')`; + } + } + return query; +} From 133de6d7925752a6947e310433437410b0e0e824 Mon Sep 17 00:00:00 2001 From: Wing-Kam Wong Date: Thu, 13 Apr 2023 16:30:29 +0800 Subject: [PATCH 5/5] refactor(nocodb): use convertDateFormatForConcat & fix sqlite case --- .../sql/formulav2/formulaQueryBuilderv2.ts | 69 +++++++++---------- 1 file changed, 31 insertions(+), 38 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index 7a660bf2b0..1ee8c93268 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -3,11 +3,13 @@ import { jsepCurlyHook, UITypes } from 'nocodb-sdk'; import mapFunctionName from '../mapFunctionName'; import genRollupSelectv2 from '../genRollupSelectv2'; import FormulaColumn from '../../../../../models/FormulaColumn'; -import { validateDateWithUnknownFormat } from '../helpers/formulaFnHelper'; +import { + convertDateFormatForConcat, + validateDateWithUnknownFormat, +} from '../helpers/formulaFnHelper'; import { CacheGetType, CacheScope } from '../../../../../utils/globals'; import NocoCache from '../../../../../cache/NocoCache'; -import Column from '../../../../../models/Column'; -import { convertDateFormat } from '../helpers/convertDateFormat'; +import type Column from '../../../../../models/Column'; import type Model from '../../../../../models/Model'; import type RollupColumn from '../../../../../models/RollupColumn'; import type { XKnex } from '../../../index'; @@ -636,38 +638,15 @@ async function _formulaQueryBuilder( pt.arguments.map(async (arg) => { let query = (await fn(arg)).builder.toQuery(); if (pt.callee.name === 'CONCAT') { - if ( - arg.type === 'Identifier' && - arg.name in columnIdToUidt && - columnIdToUidt[arg.name] === UITypes.Date - ) { - const meta = ( - await Column.get({ - colId: arg.name, - }) - ).meta; - - if (knex.clientType() === 'mysql2') { - query = `DATE_FORMAT(${query}, '${convertDateFormat( - meta.date_format, - knex.clientType() - )}')`; - } else if (knex.clientType() === 'pg') { - query = `TO_CHAR(${query}, '${convertDateFormat( - meta.date_format, - knex.clientType() - )}')`; - } else if (knex.clientType() === 'sqlite') { - query = `strftime('${convertDateFormat( - meta.date_format, - knex.clientType() - )}', ${query})`; - } else if (knex.clientType() === 'mssql') { - query = `FORMAT(${query}, '${convertDateFormat( - meta.date_format, - knex.clientType() - )}')`; - } + if (knex.clientType() !== 'sqlite3') { + query = await convertDateFormatForConcat( + arg, + columnIdToUidt, + query, + knex.clientType() + ); + } else { + // sqlite3: special handling - See BinaryExpression } if (knex.clientType() === 'mysql2') { @@ -714,8 +693,8 @@ async function _formulaQueryBuilder( pt.left.fnName = pt.left.fnName || 'ARITH'; pt.right.fnName = pt.right.fnName || 'ARITH'; - const left = (await fn(pt.left, null, pt.operator)).builder.toQuery(); - const right = (await fn(pt.right, null, pt.operator)).builder.toQuery(); + let left = (await fn(pt.left, null, pt.operator)).builder.toQuery(); + let right = (await fn(pt.right, null, pt.operator)).builder.toQuery(); let sql = `${left} ${pt.operator} ${right}${colAlias}`; // comparing a date with empty string would throw @@ -759,8 +738,22 @@ async function _formulaQueryBuilder( } } - // handle NULL values when calling CONCAT for sqlite3 if (pt.left.fnName === 'CONCAT' && knex.clientType() === 'sqlite3') { + // handle date format + left = await convertDateFormatForConcat( + pt.left?.arguments?.[0], + columnIdToUidt, + left, + knex.clientType() + ); + right = await convertDateFormatForConcat( + pt.right?.arguments?.[0], + columnIdToUidt, + right, + knex.clientType() + ); + + // handle NULL values when calling CONCAT for sqlite3 sql = `COALESCE(${left}, '') ${pt.operator} COALESCE(${right},'')${colAlias}`; }