From 33f19aa596c2d9d9f133c25c829a0231383e6c57 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 30 Mar 2023 01:09:40 +0530 Subject: [PATCH 01/27] refactor: convert to async...await to resolve builder only if required Signed-off-by: Pranav C --- .../sql/formulav2/formulaQueryBuilderv2.ts | 91 ++--- .../lib/sql/functionMappings/commonFns.ts | 56 ++- .../lib/sql/functionMappings/mssql.ts | 104 +++-- .../lib/sql/functionMappings/mysql.ts | 184 ++++----- .../lib/sql/functionMappings/pg.ts | 28 +- .../lib/sql/functionMappings/sqlite.ts | 382 +++++++++--------- .../lib/sql/mapFunctionName.ts | 9 +- 7 files changed, 450 insertions(+), 404 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index 3eb91300d9..3bd4a073d8 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -53,7 +53,7 @@ async function _formulaQueryBuilder( alias, knex: XKnex, model: Model, - aliasToColumn = {}, + aliasToColumn: Record Promise<{ builder: any }>> = {}, tableAlias?: string ) { // formula may include double curly brackets in previous version @@ -69,21 +69,25 @@ async function _formulaQueryBuilder( switch (col.uidt) { case UITypes.Formula: { - const formulOption = await col.getColOptions(); - const { builder } = await _formulaQueryBuilder( - formulOption.formula, - alias, - knex, - model, - { ...aliasToColumn, [col.id]: null }, - tableAlias - ); - builder.sql = '(' + builder.sql + ')'; - aliasToColumn[col.id] = builder; + aliasToColumn[col.id] = async () => { + const formulOption = await col.getColOptions(); + const { builder } = await _formulaQueryBuilder( + formulOption.formula, + alias, + knex, + model, + { ...aliasToColumn, [col.id]: null }, + tableAlias + ); + builder.sql = '(' + builder.sql + ')'; + return { + builder, + }; + }; } break; case UITypes.Lookup: - { + aliasToColumn[col.id] = async (): Promise => { let aliasCount = 0; let selectQb; let isMany = false; @@ -398,25 +402,27 @@ async function _formulaQueryBuilder( } if (selectQb) - aliasToColumn[col.id] = - typeof selectQb === 'function' - ? selectQb - : knex.raw(selectQb as any).wrap('(', ')'); + return { + builder: + typeof selectQb === 'function' + ? selectQb + : knex.raw(selectQb as any).wrap('(', ')'), + }; } - } + }; break; case UITypes.Rollup: - { + aliasToColumn[col.id] = async (): Promise => { const qb = await genRollupSelectv2({ knex, columnOptions: (await col.getColOptions()) as RollupColumn, alias: tableAlias, }); - aliasToColumn[col.id] = knex.raw(qb.builder).wrap('(', ')'); - } + return { builder: knex.raw(qb.builder).wrap('(', ')') }; + }; break; case UITypes.LinkToAnotherRecord: - { + aliasToColumn[col.id] = async (): Promise => { const alias = `__nc_formula_ll`; const relation = await col.getColOptions(); // if (relation.type !== 'bt') continue; @@ -520,19 +526,22 @@ async function _formulaQueryBuilder( .wrap('(', ')'); } if (selectQb) - aliasToColumn[col.id] = - typeof selectQb === 'function' - ? selectQb - : knex.raw(selectQb as any).wrap('(', ')'); - } + return { + builder: + typeof selectQb === 'function' + ? selectQb + : knex.raw(selectQb as any).wrap('(', ')'), + }; + }; break; default: - aliasToColumn[col.id] = col.column_name; + aliasToColumn[col.id] = () => + Promise.resolve({ builder: col.column_name }); break; } } - const fn = (pt, a?, prevBinaryOp?) => { + const fn = async (pt, a?, prevBinaryOp?) => { const colAlias = a ? ` as ${a}` : ''; pt.arguments?.forEach?.((arg) => { if (arg.fnName) return; @@ -558,18 +567,6 @@ async function _formulaQueryBuilder( return fn(pt.arguments[0], a, prevBinaryOp); } break; - // case 'AVG': - // if (pt.arguments.length > 1) { - // return fn({ - // type: 'BinaryExpression', - // operator: '/', - // left: {...pt, callee: {name: 'SUM'}}, - // right: {type: 'Literal', value: pt.arguments.length} - // }, a, prevBinaryOp) - // } else { - // return fn(pt.arguments[0], a, prevBinaryOp) - // } - // break; case 'CONCAT': if (knex.clientType() === 'sqlite3') { if (pt.arguments.length > 1) { @@ -653,8 +650,12 @@ async function _formulaQueryBuilder( } else if (pt.type === 'Literal') { return knex.raw(`?${colAlias}`, [pt.value]); } else if (pt.type === 'Identifier') { - if (typeof aliasToColumn?.[pt.name] === 'function') { - return knex.raw(`??${colAlias}`, aliasToColumn?.[pt.name](pt.fnName)); + const { builder } = await aliasToColumn?.[pt.name]() + if (typeof builder === 'function') { + return knex.raw( + `??${colAlias}`, + await builder(pt.fnName) + ); } return knex.raw(`??${colAlias}`, [aliasToColumn?.[pt.name] || pt.name]); } else if (pt.type === 'BinaryExpression') { @@ -772,7 +773,7 @@ async function _formulaQueryBuilder( if (prevBinaryOp && pt.operator !== prevBinaryOp) { query.wrap('(', ')'); } - return query; + return { builder: query }; } else if (pt.type === 'UnaryExpression') { const query = knex.raw( `${pt.operator}${fn( @@ -784,7 +785,7 @@ async function _formulaQueryBuilder( if (prevBinaryOp && pt.operator !== prevBinaryOp) { query.wrap('(', ')'); } - return query; + return { builder: query }; } }; return { builder: fn(tree, alias) }; diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts index 2b65d2cace..1fac4fbbd7 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts @@ -2,66 +2,80 @@ import type { MapFnArgs } from '../mapFunctionName'; export default { // todo: handle default case - SWITCH: (args: MapFnArgs) => { + SWITCH: async (args: MapFnArgs) => { const count = Math.floor((args.pt.arguments.length - 1) / 2); let query = ''; - const switchVal = args.fn(args.pt.arguments[0]).toQuery(); + const switchVal = (await args.fn(args.pt.arguments[0])).builder.toQuery(); for (let i = 0; i < count; i++) { query += args.knex .raw( - `\n\tWHEN ${args - .fn(args.pt.arguments[i * 2 + 1]) - .toQuery()} THEN ${args.fn(args.pt.arguments[i * 2 + 2]).toQuery()}` + `\n\tWHEN ${( + await args.fn(args.pt.arguments[i * 2 + 1]) + ).builder.toQuery()} THEN ${( + await args.fn(args.pt.arguments[i * 2 + 2]) + ).builder.toQuery()}` ) .toQuery(); } if (args.pt.arguments.length % 2 === 0) { query += args.knex .raw( - `\n\tELSE ${args - .fn(args.pt.arguments[args.pt.arguments.length - 1]) - .toQuery()}` + `\n\tELSE ${( + await args.fn(args.pt.arguments[args.pt.arguments.length - 1]) + ).builder.toQuery()}` ) .toQuery(); } return args.knex.raw(`CASE ${switchVal} ${query}\n END${args.colAlias}`); }, - IF: (args: MapFnArgs) => { + IF: async (args: MapFnArgs) => { let query = args.knex .raw( - `\n\tWHEN ${args.fn(args.pt.arguments[0]).toQuery()} THEN ${args - .fn(args.pt.arguments[1]) - .toQuery()}` + `\n\tWHEN ${( + await args.fn(args.pt.arguments[0]) + ).builder.toQuery()} THEN ${( + await args.fn(args.pt.arguments[1]) + ).builder.toQuery()}` ) .toQuery(); if (args.pt.arguments[2]) { query += args.knex - .raw(`\n\tELSE ${args.fn(args.pt.arguments[2]).toQuery()}`) + .raw(`\n\tELSE ${(await args.fn(args.pt.arguments[2])).builder.toQuery()}`) .toQuery(); } return args.knex.raw(`CASE ${query}\n END${args.colAlias}`); }, TRUE: (_args) => 1, FALSE: (_args) => 0, - AND: (args: MapFnArgs) => { + AND: async (args: MapFnArgs) => { return args.knex.raw( `${args.knex .raw( - `${args.pt.arguments - .map((ar) => args.fn(ar).toQuery()) - .join(' AND ')}` + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar)).builder.toQuery() + ) + ) + ).join(' AND ')}` ) .wrap('(', ')') .toQuery()}${args.colAlias}` ); }, - OR: (args: MapFnArgs) => { + OR: async (args: MapFnArgs) => { return args.knex.raw( `${args.knex .raw( - `${args.pt.arguments.map((ar) => args.fn(ar).toQuery()).join(' OR ')}` + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar)).builder.toQuery() + ) + ) + ).join(' OR ')}` ) .wrap('(', ')') .toQuery()}${args.colAlias}` @@ -83,7 +97,7 @@ export default { return args.fn(args.pt.arguments[0], args.a, args.prevBinaryOp); } }, - FLOAT: (args: MapFnArgs) => { - return args.fn(args.pt?.arguments?.[0]).wrap('(', ')'); + FLOAT: async (args: MapFnArgs) => { + return (await args.fn(args.pt?.arguments?.[0])).builder.wrap('(', ')'); }, }; diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts index 105311cade..6bf5e41831 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts @@ -6,48 +6,61 @@ import type { MapFnArgs } from '../mapFunctionName'; const mssql = { ...commonFns, - MIN: (args: MapFnArgs) => { + MIN: async (args: MapFnArgs) => { if (args.pt.arguments.length === 1) { return args.fn(args.pt.arguments[0]); } let query = ''; for (const [i, arg] of Object.entries(args.pt.arguments)) { if (+i === args.pt.arguments.length - 1) { - query += args.knex.raw(`\n\tElse ${args.fn(arg).toQuery()}`).toQuery(); + query += args.knex + .raw( + `\n\tElse ${(await await args.fn(arg)).builder.builder.toQuery()}` + ) + .toQuery(); } else { query += args.knex .raw( - `\n\tWhen ${args.pt.arguments - .filter((_, j) => +i !== j) - .map( - (arg1) => - `${args.fn(arg).toQuery()} < ${args.fn(arg1).toQuery()}` + `\n\tWhen ${( + await Promise.all( + args.pt.arguments + .filter((_, j) => +i !== j) + .map( + async (arg1) => + `${(await args.fn(arg)).builder.toQuery()} < ${( + await args.fn(arg1) + ).builder.toQuery()}` + ) ) - .join(' And ')} Then ${args.fn(arg).toQuery()}` + ).join(' And ')} Then ${(await args.fn(arg)).builder.toQuery()}` ) .toQuery(); } } return args.knex.raw(`Case ${query}\n End${args.colAlias}`); }, - MAX: (args: MapFnArgs) => { + MAX: async (args: MapFnArgs) => { if (args.pt.arguments.length === 1) { return args.fn(args.pt.arguments[0]); } let query = ''; for (const [i, arg] of Object.entries(args.pt.arguments)) { if (+i === args.pt.arguments.length - 1) { - query += args.knex.raw(`\nElse ${args.fn(arg).toQuery()}`).toQuery(); + query += args.knex + .raw(`\nElse ${(await args.fn(arg)).builder.toQuery()}`) + .toQuery(); } else { query += args.knex .raw( `\nWhen ${args.pt.arguments .filter((_, j) => +i !== j) .map( - (arg1) => - `${args.fn(arg).toQuery()} > ${args.fn(arg1).toQuery()}` + async (arg1) => + `${(await args.fn(arg)).builder.toQuery()} > ${( + await args.fn(arg1) + ).builder.toQuery()}` ) - .join(' And ')} Then ${args.fn(arg).toQuery()}` + .join(' And ')} Then ${(await args.fn(arg)).builder.toQuery()}` ) .toQuery(); } @@ -55,12 +68,15 @@ const mssql = { return args.knex.raw(`Case ${query}\n End${args.colAlias}`); }, - LOG: (args: MapFnArgs) => { + LOG: async (args: MapFnArgs) => { return args.knex.raw( - `LOG(${args.pt.arguments - .reverse() - .map((ar) => args.fn(ar).toQuery()) - .join(',')})${args.colAlias}` + `LOG(${( + await Promise.all( + args.pt.arguments + .reverse() + .map(async (ar) => (await args.fn(ar)).builder.toQuery()) + ) + ).join(',')})${args.colAlias}` ); }, MOD: (pt) => { @@ -73,19 +89,19 @@ const mssql = { }, REPEAT: 'REPLICATE', NOW: 'getdate', - SEARCH: (args: MapFnArgs) => { + SEARCH: async (args: MapFnArgs) => { args.pt.callee.name = 'CHARINDEX'; const temp = args.pt.arguments[0]; args.pt.arguments[0] = args.pt.arguments[1]; args.pt.arguments[1] = temp; }, - INT: (args: MapFnArgs) => { + INT: async (args: MapFnArgs) => { return args.knex.raw( - `CASE WHEN ISNUMERIC(${args - .fn(args.pt.arguments[0]) - .toQuery()}) = 1 THEN FLOOR(${args - .fn(args.pt.arguments[0]) - .toQuery()}) ELSE 0 END${args.colAlias}` + `CASE WHEN ISNUMERIC(${( + await args.fn(args.pt.arguments[0]) + ).builder.toQuery()}) = 1 THEN FLOOR(${( + await args.fn(args.pt.arguments[0]) + ).builder.toQuery()}) ELSE 0 END${args.colAlias}` ); }, MID: 'SUBSTR', @@ -94,8 +110,8 @@ const mssql = { .raw(`CAST(${args.fn(args.pt.arguments[0])} as FLOAT)${args.colAlias}`) .wrap('(', ')'); }, - DATEADD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - const dateIN = fn(pt.arguments[1]); + DATEADD: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + const dateIN = (await fn(pt.arguments[1])).builder; return knex.raw( `CASE WHEN ${fn(pt.arguments[0])} LIKE '%:%' THEN @@ -111,49 +127,59 @@ const mssql = { END${colAlias}` ); }, - DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + DATETIME_DIFF: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { const datetime_expr1 = fn(pt.arguments[0]); const datetime_expr2 = fn(pt.arguments[1]); const rawUnit = pt.arguments[2] - ? fn(pt.arguments[2]).bindings[0] + ? (await fn(pt.arguments[2])).builder.bindings[0] : 'seconds'; const unit = convertUnits(rawUnit, 'mssql'); return knex.raw( `DATEDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}` ); }, - WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + WEEKDAY: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { // DATEPART(WEEKDAY, DATE): sunday = 1, monday = 2, ..., saturday = 7 // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday return knex.raw( `(DATEPART(WEEKDAY, ${ pt.arguments[0].type === 'Literal' - ? `'${dayjs(fn(pt.arguments[0])).format('YYYY-MM-DD')}'` + ? `'${dayjs((await fn(pt.arguments[0])).builder).format( + 'YYYY-MM-DD' + )}'` : fn(pt.arguments[0]) }) - 2 - ${getWeekdayByText( pt?.arguments[1]?.value )} % 7 + 7) % 7 ${colAlias}` ); }, - AND: (args: MapFnArgs) => { + AND: async (args: MapFnArgs) => { return args.knex.raw( `CASE WHEN ${args.knex .raw( - `${args.pt.arguments - .map((ar) => args.fn(ar, '', 'AND').toQuery()) - .join(' AND ')}` + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar, '', 'AND')).builder.toQuery() + ) + ) + ).join(' AND ')}` ) .wrap('(', ')') .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` ); }, - OR: (args: MapFnArgs) => { + OR: async (args: MapFnArgs) => { return args.knex.raw( `CASE WHEN ${args.knex .raw( - `${args.pt.arguments - .map((ar) => args.fn(ar, '', 'OR').toQuery()) - .join(' OR ')}` + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar, '', 'OR')).builder.toQuery() + ) + ) + ).join(' OR ')}` ) .wrap('(', ')') .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts index eae5c0f3f1..1c10b63434 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts @@ -1,99 +1,99 @@ -import dayjs from 'dayjs'; -import { convertUnits } from '../helpers/convertUnits'; -import { getWeekdayByText } from '../helpers/formulaFnHelper'; +// import dayjs from 'dayjs'; +// import { convertUnits } from '../helpers/convertUnits'; +// import { getWeekdayByText } from '../helpers/formulaFnHelper'; import commonFns from './commonFns'; -import type { MapFnArgs } from '../mapFunctionName'; +// import type { MapFnArgs } from '../mapFunctionName'; const mysql2 = { ...commonFns, - LEN: 'CHAR_LENGTH', - MIN: 'LEAST', - MAX: 'GREATEST', - SEARCH: (args: MapFnArgs) => { - args.pt.callee.name = 'LOCATE'; - const temp = args.pt.arguments[0]; - args.pt.arguments[0] = args.pt.arguments[1]; - args.pt.arguments[1] = temp; - }, - INT: (args: MapFnArgs) => { - return args.knex.raw( - `CAST(${args.fn(args.pt.arguments[0])} as SIGNED)${args.colAlias}` - ); - }, - LEFT: (args: MapFnArgs) => { - return args.knex.raw( - `SUBSTR(${args.fn(args.pt.arguments[0])},1,${args.fn( - args.pt.arguments[1] - )})${args.colAlias}` - ); - }, - RIGHT: (args: MapFnArgs) => { - return args.knex.raw( - `SUBSTR(${args.fn(args.pt.arguments[0])}, -(${args.fn( - args.pt.arguments[1] - )}))${args.colAlias}` - ); - }, - MID: 'SUBSTR', - FLOAT: (args: MapFnArgs) => { - return args.knex - .raw( - `CAST(CAST(${args.fn(args.pt.arguments[0])} as CHAR) AS DOUBLE)${ - args.colAlias - }` - ) - .wrap('(', ')'); - }, - DATEADD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - return knex.raw( - `CASE - WHEN ${fn(pt.arguments[0])} LIKE '%:%' THEN - DATE_FORMAT(DATE_ADD(${fn(pt.arguments[0])}, INTERVAL - ${fn(pt.arguments[1])} ${String(fn(pt.arguments[2])).replace( - /["']/g, - '' - )}), '%Y-%m-%d %H:%i') - ELSE - DATE(DATE_ADD(${fn(pt.arguments[0])}, INTERVAL - ${fn(pt.arguments[1])} ${String(fn(pt.arguments[2])).replace( - /["']/g, - '' - )})) - END${colAlias}` - ); - }, - DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - const datetime_expr1 = fn(pt.arguments[0]); - const datetime_expr2 = fn(pt.arguments[1]); - - const unit = convertUnits( - pt.arguments[2] ? fn(pt.arguments[2]).bindings[0] : 'seconds', - 'mysql' - ); - - if (unit === 'MICROSECOND') { - // MySQL doesn't support millisecond - // hence change from MICROSECOND to millisecond manually - return knex.raw( - `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) div 1000 ${colAlias}` - ); - } - return knex.raw( - `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}` - ); - }, - WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday - return knex.raw( - `(WEEKDAY(${ - pt.arguments[0].type === 'Literal' - ? `'${dayjs(fn(pt.arguments[0])).format('YYYY-MM-DD')}'` - : fn(pt.arguments[0]) - }) - ${getWeekdayByText( - pt?.arguments[1]?.value - )} % 7 + 7) % 7 ${colAlias}` - ); - }, + // LEN: 'CHAR_LENGTH', + // MIN: 'LEAST', + // MAX: 'GREATEST', + // SEARCH: (args: MapFnArgs) => { + // args.pt.callee.name = 'LOCATE'; + // const temp = args.pt.arguments[0]; + // args.pt.arguments[0] = args.pt.arguments[1]; + // args.pt.arguments[1] = temp; + // }, + // INT: (args: MapFnArgs) => { + // return args.knex.raw( + // `CAST(${args.fn(args.pt.arguments[0])} as SIGNED)${args.colAlias}` + // ); + // }, + // LEFT: (args: MapFnArgs) => { + // return args.knex.raw( + // `SUBSTR(${args.fn(args.pt.arguments[0])},1,${args.fn( + // args.pt.arguments[1] + // )})${args.colAlias}` + // ); + // }, + // RIGHT: (args: MapFnArgs) => { + // return args.knex.raw( + // `SUBSTR(${args.fn(args.pt.arguments[0])}, -(${args.fn( + // args.pt.arguments[1] + // )}))${args.colAlias}` + // ); + // }, + // MID: 'SUBSTR', + // FLOAT: (args: MapFnArgs) => { + // return args.knex + // .raw( + // `CAST(CAST(${args.fn(args.pt.arguments[0])} as CHAR) AS DOUBLE)${ + // args.colAlias + // }` + // ) + // .wrap('(', ')'); + // }, + // DATEADD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + // return knex.raw( + // `CASE + // WHEN ${fn(pt.arguments[0])} LIKE '%:%' THEN + // DATE_FORMAT(DATE_ADD(${fn(pt.arguments[0])}, INTERVAL + // ${fn(pt.arguments[1])} ${String(fn(pt.arguments[2])).replace( + // /["']/g, + // '' + // )}), '%Y-%m-%d %H:%i') + // ELSE + // DATE(DATE_ADD(${fn(pt.arguments[0])}, INTERVAL + // ${fn(pt.arguments[1])} ${String(fn(pt.arguments[2])).replace( + // /["']/g, + // '' + // )})) + // END${colAlias}` + // ); + // }, + // DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + // const datetime_expr1 = fn(pt.arguments[0]); + // const datetime_expr2 = fn(pt.arguments[1]); + // + // const unit = convertUnits( + // pt.arguments[2] ? fn(pt.arguments[2]).bindings[0] : 'seconds', + // 'mysql' + // ); + // + // if (unit === 'MICROSECOND') { + // // MySQL doesn't support millisecond + // // hence change from MICROSECOND to millisecond manually + // return knex.raw( + // `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) div 1000 ${colAlias}` + // ); + // } + // return knex.raw( + // `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}` + // ); + // }, + // WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + // // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday + // return knex.raw( + // `(WEEKDAY(${ + // pt.arguments[0].type === 'Literal' + // ? `'${dayjs(fn(pt.arguments[0])).format('YYYY-MM-DD')}'` + // : fn(pt.arguments[0]) + // }) - ${getWeekdayByText( + // pt?.arguments[1]?.value + // )} % 7 + 7) % 7 ${colAlias}` + // ); + // }, }; export default mysql2; diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts index 0efcfc9cb3..b361996859 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts @@ -12,13 +12,13 @@ const pg = { CEILING: 'ceil', POWER: 'pow', SQRT: 'sqrt', - SEARCH: (args: MapFnArgs) => { + SEARCH: async (args: MapFnArgs) => { return args.knex.raw( `POSITION(${args.knex.raw( - args.fn(args.pt.arguments[1]).toQuery() - )} in ${args.knex.raw(args.fn(args.pt.arguments[0]).toQuery())})${ - args.colAlias - }` + (await args.fn(args.pt.arguments[1])).builder.toQuery() + )} in ${args.knex + .raw((await args.fn(args.pt.arguments[0])).builder) + .toQuery()})${args.colAlias}` ); }, INT(args: MapFnArgs) { @@ -51,11 +51,11 @@ const pg = { )}')::interval${colAlias}` ); }, - DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + DATETIME_DIFF: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { const datetime_expr1 = fn(pt.arguments[0]); const datetime_expr2 = fn(pt.arguments[1]); const rawUnit = pt.arguments[2] - ? fn(pt.arguments[2]).bindings[0] + ? (await fn(pt.arguments[2])).builder.bindings[0] : 'seconds'; let sql; const unit = convertUnits(rawUnit, 'pg'); @@ -101,37 +101,39 @@ const pg = { } return knex.raw(`${sql} ${colAlias}`); }, - WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + WEEKDAY: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { // isodow: the day of the week as Monday (1) to Sunday (7) // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday return knex.raw( `(EXTRACT(ISODOW FROM ${ pt.arguments[0].type === 'Literal' - ? `date '${dayjs(fn(pt.arguments[0])).format('YYYY-MM-DD')}'` + ? `date '${dayjs((await fn(pt.arguments[0])).builder).format( + 'YYYY-MM-DD' + )}'` : fn(pt.arguments[0]) }) - 1 - ${getWeekdayByText( pt?.arguments[1]?.value )} % 7 + 7) ::INTEGER % 7 ${colAlias}` ); }, - AND: (args: MapFnArgs) => { + AND: async (args: MapFnArgs) => { return args.knex.raw( `CASE WHEN ${args.knex .raw( `${args.pt.arguments - .map((ar) => args.fn(ar, '', 'AND').toQuery()) + .map(async (ar) => (await args.fn(ar, '', 'AND')).builder.toQuery()) .join(' AND ')}` ) .wrap('(', ')') .toQuery()} THEN TRUE ELSE FALSE END ${args.colAlias}` ); }, - OR: (args: MapFnArgs) => { + OR: async (args: MapFnArgs) => { return args.knex.raw( `CASE WHEN ${args.knex .raw( `${args.pt.arguments - .map((ar) => args.fn(ar, '', 'OR').toQuery()) + .map(async (ar) => (await args.fn(ar, '', 'OR')).builder.toQuery()) .join(' OR ')}` ) .wrap('(', ')') diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts index d4bfb56d82..187c8cf156 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts @@ -1,198 +1,198 @@ -import dayjs from 'dayjs'; -import { convertUnits } from '../helpers/convertUnits'; -import { getWeekdayByText } from '../helpers/formulaFnHelper'; -import { - convertToTargetFormat, - getDateFormat, -} from '../../../../../utils/dateTimeUtils'; +// import dayjs from 'dayjs'; +// import { convertUnits } from '../helpers/convertUnits'; +// import { getWeekdayByText } from '../helpers/formulaFnHelper'; +// import { +// convertToTargetFormat, +// getDateFormat, +// } from '../../../../../utils/dateTimeUtils'; import commonFns from './commonFns'; -import type { MapFnArgs } from '../mapFunctionName'; +// import type { MapFnArgs } from '../mapFunctionName'; const sqlite3 = { ...commonFns, - LEN: 'LENGTH', - CEILING(args) { - return args.knex.raw( - `round(${args.fn(args.pt.arguments[0])} + 0.5)${args.colAlias}` - ); - }, - FLOOR(args) { - return args.knex.raw( - `round(${args.fn(args.pt.arguments[0])} - 0.5)${args.colAlias}` - ); - }, - MOD: (args: MapFnArgs) => { - return args.fn({ - type: 'BinaryExpression', - operator: '%', - left: args.pt.arguments[0], - right: args.pt.arguments[1], - }); - }, - REPEAT(args: MapFnArgs) { - return args.knex.raw( - `replace(printf('%.' || ${args.fn( - args.pt.arguments[1] - )} || 'c', '/'),'/',${args.fn(args.pt.arguments[0])})${args.colAlias}` - ); - }, - NOW: 'DATE', - SEARCH: 'INSTR', - INT(args: MapFnArgs) { - return args.knex.raw( - `CAST(${args.fn(args.pt.arguments[0])} as INTEGER)${args.colAlias}` - ); - }, - LEFT: (args: MapFnArgs) => { - return args.knex.raw( - `SUBSTR(${args.fn(args.pt.arguments[0])},1,${args.fn( - args.pt.arguments[1] - )})${args.colAlias}` - ); - }, - RIGHT: (args: MapFnArgs) => { - return args.knex.raw( - `SUBSTR(${args.fn(args.pt.arguments[0])},-(${args.fn( - args.pt.arguments[1] - )}))${args.colAlias}` - ); - }, - MID: 'SUBSTR', - FLOAT: (args: MapFnArgs) => { - return args.knex - .raw(`CAST(${args.fn(args.pt.arguments[0])} as FLOAT)${args.colAlias}`) - .wrap('(', ')'); - }, - DATEADD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - const dateIN = fn(pt.arguments[1]); - return knex.raw( - `CASE - WHEN ${fn(pt.arguments[0])} LIKE '%:%' THEN - STRFTIME('%Y-%m-%d %H:%M', DATETIME(DATETIME(${fn( - pt.arguments[0] - )}, 'localtime'), - ${dateIN > 0 ? '+' : ''}${fn(pt.arguments[1])} || ' ${String( - fn(pt.arguments[2]) - ).replace(/["']/g, '')}')) - ELSE - DATE(DATETIME(${fn(pt.arguments[0])}, 'localtime'), - ${dateIN > 0 ? '+' : ''}${fn(pt.arguments[1])} || ' ${String( - fn(pt.arguments[2]) - ).replace(/["']/g, '')}') - END${colAlias}` - ); - }, - DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - let datetime_expr1 = fn(pt.arguments[0]); - let datetime_expr2 = fn(pt.arguments[1]); - // JULIANDAY takes YYYY-MM-DD - if (datetime_expr1.sql === '?' && datetime_expr1.bindings?.[0]) { - datetime_expr1 = `'${convertToTargetFormat( - datetime_expr1.bindings[0], - getDateFormat(datetime_expr1.bindings[0]), - 'YYYY-MM-DD' - )}'`; - } - - if (datetime_expr2.sql === '?' && datetime_expr2.bindings?.[0]) { - datetime_expr2 = `'${convertToTargetFormat( - datetime_expr2.bindings[0], - getDateFormat(datetime_expr2.bindings[0]), - 'YYYY-MM-DD' - )}'`; - } - - const rawUnit = pt.arguments[2] - ? fn(pt.arguments[2]).bindings[0] - : 'seconds'; - let sql; - const unit = convertUnits(rawUnit, 'sqlite'); - switch (unit) { - case 'seconds': - sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2}))`; - break; - case 'minutes': - sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) / 60`; - break; - case 'hours': - sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) / 3600`; - break; - case 'milliseconds': - sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) * 1000`; - break; - case 'weeks': - sql = `ROUND((JULIANDAY(${datetime_expr1}) - JULIANDAY(${datetime_expr2})) / 7)`; - break; - case 'months': - sql = `(strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) * 12 + (strftime('%m', ${datetime_expr1}) - strftime('%m', ${datetime_expr2})) `; - break; - case 'quarters': - sql = `(strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) * 4 + (strftime('%m', ${datetime_expr1}) - strftime('%m', ${datetime_expr2})) / 3`; - break; - case 'years': - sql = `CASE - WHEN (${datetime_expr2} < ${datetime_expr1}) THEN - ( - (strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) - - (strftime('%m', ${datetime_expr1}) < strftime('%m', ${datetime_expr2}) - OR (strftime('%m', ${datetime_expr1}) = strftime('%m', ${datetime_expr2}) - AND strftime('%d', ${datetime_expr1}) < strftime('%d', ${datetime_expr2}))) - ) - WHEN (${datetime_expr2} > ${datetime_expr1}) THEN - -1 * ( - (strftime('%Y', ${datetime_expr2}) - strftime('%Y', ${datetime_expr1})) - - (strftime('%m', ${datetime_expr2}) < strftime('%m', ${datetime_expr1}) - OR (strftime('%m', ${datetime_expr2}) = strftime('%m', ${datetime_expr1}) - AND strftime('%d', ${datetime_expr2}) < strftime('%d', ${datetime_expr1}))) - ) - ELSE 0 - END`; - break; - case 'days': - sql = `JULIANDAY(${datetime_expr1}) - JULIANDAY(${datetime_expr2})`; - break; - default: - sql = ''; - } - return knex.raw(`${sql} ${colAlias}`); - }, - WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - // strftime('%w', date) - day of week 0 - 6 with Sunday == 0 - // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday - return knex.raw( - `(strftime('%w', ${ - pt.arguments[0].type === 'Literal' - ? `'${dayjs(fn(pt.arguments[0])).format('YYYY-MM-DD')}'` - : fn(pt.arguments[0]) - }) - 1 - ${getWeekdayByText( - pt?.arguments[1]?.value - )} % 7 + 7) % 7 ${colAlias}` - ); - }, - AND: (args: MapFnArgs) => { - return args.knex.raw( - `CASE WHEN ${args.knex - .raw( - `${args.pt.arguments - .map((ar) => args.fn(ar, '', 'AND').toQuery()) - .join(' AND ')}` - ) - .wrap('(', ')') - .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` - ); - }, - OR: (args: MapFnArgs) => { - return args.knex.raw( - `CASE WHEN ${args.knex - .raw( - `${args.pt.arguments - .map((ar) => args.fn(ar, '', 'OR').toQuery()) - .join(' OR ')}` - ) - .wrap('(', ')') - .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` - ); - }, + // LEN: 'LENGTH', + // CEILING(args) { + // return args.knex.raw( + // `round(${args.fn(args.pt.arguments[0])} + 0.5)${args.colAlias}` + // ); + // }, + // FLOOR(args) { + // return args.knex.raw( + // `round(${args.fn(args.pt.arguments[0])} - 0.5)${args.colAlias}` + // ); + // }, + // MOD: (args: MapFnArgs) => { + // return args.fn({ + // type: 'BinaryExpression', + // operator: '%', + // left: args.pt.arguments[0], + // right: args.pt.arguments[1], + // }); + // }, + // REPEAT(args: MapFnArgs) { + // return args.knex.raw( + // `replace(printf('%.' || ${args.fn( + // args.pt.arguments[1] + // )} || 'c', '/'),'/',${args.fn(args.pt.arguments[0])})${args.colAlias}` + // ); + // }, + // NOW: 'DATE', + // SEARCH: 'INSTR', + // INT(args: MapFnArgs) { + // return args.knex.raw( + // `CAST(${args.fn(args.pt.arguments[0])} as INTEGER)${args.colAlias}` + // ); + // }, + // LEFT: (args: MapFnArgs) => { + // return args.knex.raw( + // `SUBSTR(${args.fn(args.pt.arguments[0])},1,${args.fn( + // args.pt.arguments[1] + // )})${args.colAlias}` + // ); + // }, + // RIGHT: (args: MapFnArgs) => { + // return args.knex.raw( + // `SUBSTR(${args.fn(args.pt.arguments[0])},-(${args.fn( + // args.pt.arguments[1] + // )}))${args.colAlias}` + // ); + // }, + // MID: 'SUBSTR', + // FLOAT: (args: MapFnArgs) => { + // return args.knex + // .raw(`CAST(${args.fn(args.pt.arguments[0])} as FLOAT)${args.colAlias}`) + // .wrap('(', ')'); + // }, + // DATEADD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + // const dateIN = fn(pt.arguments[1]); + // return knex.raw( + // `CASE + // WHEN ${fn(pt.arguments[0])} LIKE '%:%' THEN + // STRFTIME('%Y-%m-%d %H:%M', DATETIME(DATETIME(${fn( + // pt.arguments[0] + // )}, 'localtime'), + // ${dateIN > 0 ? '+' : ''}${fn(pt.arguments[1])} || ' ${String( + // fn(pt.arguments[2]) + // ).replace(/["']/g, '')}')) + // ELSE + // DATE(DATETIME(${fn(pt.arguments[0])}, 'localtime'), + // ${dateIN > 0 ? '+' : ''}${fn(pt.arguments[1])} || ' ${String( + // fn(pt.arguments[2]) + // ).replace(/["']/g, '')}') + // END${colAlias}` + // ); + // }, + // DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + // let datetime_expr1 = fn(pt.arguments[0]); + // let datetime_expr2 = fn(pt.arguments[1]); + // // JULIANDAY takes YYYY-MM-DD + // if (datetime_expr1.sql === '?' && datetime_expr1.bindings?.[0]) { + // datetime_expr1 = `'${convertToTargetFormat( + // datetime_expr1.bindings[0], + // getDateFormat(datetime_expr1.bindings[0]), + // 'YYYY-MM-DD' + // )}'`; + // } + // + // if (datetime_expr2.sql === '?' && datetime_expr2.bindings?.[0]) { + // datetime_expr2 = `'${convertToTargetFormat( + // datetime_expr2.bindings[0], + // getDateFormat(datetime_expr2.bindings[0]), + // 'YYYY-MM-DD' + // )}'`; + // } + // + // const rawUnit = pt.arguments[2] + // ? fn(pt.arguments[2]).bindings[0] + // : 'seconds'; + // let sql; + // const unit = convertUnits(rawUnit, 'sqlite'); + // switch (unit) { + // case 'seconds': + // sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2}))`; + // break; + // case 'minutes': + // sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) / 60`; + // break; + // case 'hours': + // sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) / 3600`; + // break; + // case 'milliseconds': + // sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) * 1000`; + // break; + // case 'weeks': + // sql = `ROUND((JULIANDAY(${datetime_expr1}) - JULIANDAY(${datetime_expr2})) / 7)`; + // break; + // case 'months': + // sql = `(strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) * 12 + (strftime('%m', ${datetime_expr1}) - strftime('%m', ${datetime_expr2})) `; + // break; + // case 'quarters': + // sql = `(strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) * 4 + (strftime('%m', ${datetime_expr1}) - strftime('%m', ${datetime_expr2})) / 3`; + // break; + // case 'years': + // sql = `CASE + // WHEN (${datetime_expr2} < ${datetime_expr1}) THEN + // ( + // (strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) + // - (strftime('%m', ${datetime_expr1}) < strftime('%m', ${datetime_expr2}) + // OR (strftime('%m', ${datetime_expr1}) = strftime('%m', ${datetime_expr2}) + // AND strftime('%d', ${datetime_expr1}) < strftime('%d', ${datetime_expr2}))) + // ) + // WHEN (${datetime_expr2} > ${datetime_expr1}) THEN + // -1 * ( + // (strftime('%Y', ${datetime_expr2}) - strftime('%Y', ${datetime_expr1})) + // - (strftime('%m', ${datetime_expr2}) < strftime('%m', ${datetime_expr1}) + // OR (strftime('%m', ${datetime_expr2}) = strftime('%m', ${datetime_expr1}) + // AND strftime('%d', ${datetime_expr2}) < strftime('%d', ${datetime_expr1}))) + // ) + // ELSE 0 + // END`; + // break; + // case 'days': + // sql = `JULIANDAY(${datetime_expr1}) - JULIANDAY(${datetime_expr2})`; + // break; + // default: + // sql = ''; + // } + // return knex.raw(`${sql} ${colAlias}`); + // }, + // WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + // // strftime('%w', date) - day of week 0 - 6 with Sunday == 0 + // // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday + // return knex.raw( + // `(strftime('%w', ${ + // pt.arguments[0].type === 'Literal' + // ? `'${dayjs(fn(pt.arguments[0])).format('YYYY-MM-DD')}'` + // : fn(pt.arguments[0]) + // }) - 1 - ${getWeekdayByText( + // pt?.arguments[1]?.value + // )} % 7 + 7) % 7 ${colAlias}` + // ); + // }, + // AND: (args: MapFnArgs) => { + // return args.knex.raw( + // `CASE WHEN ${args.knex + // .raw( + // `${args.pt.arguments + // .map((ar) => args.fn(ar, '', 'AND').toQuery()) + // .join(' AND ')}` + // ) + // .wrap('(', ')') + // .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` + // ); + // }, + // OR: (args: MapFnArgs) => { + // return args.knex.raw( + // `CASE WHEN ${args.knex + // .raw( + // `${args.pt.arguments + // .map((ar) => args.fn(ar, '', 'OR').toQuery()) + // .join(' OR ')}` + // ) + // .wrap('(', ')') + // .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` + // ); + // }, }; export default sqlite3; diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/mapFunctionName.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/mapFunctionName.ts index 49abd50ac2..001e82fa26 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/mapFunctionName.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/mapFunctionName.ts @@ -7,16 +7,19 @@ import type { Knex } from 'knex'; export interface MapFnArgs { pt: any; - aliasToCol: { [alias: string]: string }; + aliasToCol: Record< + string, + (() => Promise<{ builder: any }>) | string | undefined + >; knex: XKnex; alias: string; a?: string; - fn: (...args: any) => Knex.QueryBuilder | any; + fn: (...args: any) => Promise<{ builder: Knex.QueryBuilder | any }>; colAlias: string; prevBinaryOp?: any; } -const mapFunctionName = (args: MapFnArgs): any => { +const mapFunctionName = async (args: MapFnArgs): Promise => { const name = args.pt.callee.name; let val; From ec150b1a5a76e7239a5466cc2db4bce26ae20206 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 30 Mar 2023 18:31:59 +0530 Subject: [PATCH 02/27] fix(nocodb): corrections Signed-off-by: Pranav C --- .../sql/formulav2/formulaQueryBuilderv2.ts | 116 ++++++++++---- .../lib/sql/functionMappings/commonFns.ts | 102 ++++++------ .../lib/sql/functionMappings/pg.ts | 146 ++++++++++-------- 3 files changed, 232 insertions(+), 132 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index 3bd4a073d8..a3910464ef 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -628,36 +628,35 @@ async function _formulaQueryBuilder( break; } - return knex.raw( - `${pt.callee.name}(${pt.arguments - .map((arg) => { - const query = fn(arg).toQuery(); - if (pt.callee.name === 'CONCAT') { - if (knex.clientType() === 'mysql2') { - // mysql2: CONCAT() returns NULL if any argument is NULL. - // adding IFNULL to convert NULL values to empty strings - return `IFNULL(${query}, '')`; - } else { - // do nothing - // pg / mssql: Concatenate all arguments. NULL arguments are ignored. - // sqlite3: special handling - See BinaryExpression + return { + builder: knex.raw( + `${pt.callee.name}(${pt.arguments + .map((arg) => { + const query = fn(arg).toQuery(); + if (pt.callee.name === 'CONCAT') { + if (knex.clientType() === 'mysql2') { + // mysql2: CONCAT() returns NULL if any argument is NULL. + // adding IFNULL to convert NULL values to empty strings + return `IFNULL(${query}, '')`; + } else { + // do nothing + // pg / mssql: Concatenate all arguments. NULL arguments are ignored. + // sqlite3: special handling - See BinaryExpression + } } - } - return query; - }) - .join()})${colAlias}`.replace(/\?/g, '\\?') - ); + return query; + }) + .join()})${colAlias}`.replace(/\?/g, '\\?') + ), + }; } else if (pt.type === 'Literal') { - return knex.raw(`?${colAlias}`, [pt.value]); + return { builder: knex.raw(`?${colAlias}`, [pt.value]) }; } else if (pt.type === 'Identifier') { - const { builder } = await aliasToColumn?.[pt.name]() + const { builder } = await aliasToColumn?.[pt.name]?.(); if (typeof builder === 'function') { - return knex.raw( - `??${colAlias}`, - await builder(pt.fnName) - ); + return { builder: knex.raw(`??${colAlias}`, await builder(pt.fnName)) }; } - return knex.raw(`??${colAlias}`, [aliasToColumn?.[pt.name] || pt.name]); + return { builder: knex.raw(`??${colAlias}`, [builder || pt.name]) }; } else if (pt.type === 'BinaryExpression') { if (pt.operator === '==') { pt.operator = '='; @@ -678,8 +677,8 @@ async function _formulaQueryBuilder( pt.left.fnName = pt.left.fnName || 'ARITH'; pt.right.fnName = pt.right.fnName || 'ARITH'; - const left = fn(pt.left, null, pt.operator).toQuery(); - const right = fn(pt.right, null, pt.operator).toQuery(); + const left = (await fn(pt.left, null, pt.operator)).builder.toQuery(); + const right = (await fn(pt.right, null, pt.operator)).builder.toQuery(); let sql = `${left} ${pt.operator} ${right}${colAlias}`; // comparing a date with empty string would throw @@ -788,7 +787,9 @@ async function _formulaQueryBuilder( return { builder: query }; } }; - return { builder: fn(tree, alias) }; + const builder = (await fn(tree, alias)).builder; + + return { builder }; } function getTnPath(tb: Model, knex, tableAlias?: string) { @@ -877,3 +878,62 @@ export default async function formulaQueryBuilderv2( } return qb; } + +export async function validateFormula( + _tree, + alias, + knex: XKnex, + model: Model, + column?: Column, + aliasToColumn = {}, + tableAlias?: string +) { + // register jsep curly hook once only + jsep.plugins.register(jsepCurlyHook); + // generate qb + const qb = await _formulaQueryBuilder( + _tree, + alias, + knex, + model, + aliasToColumn, + tableAlias + ); + + try { + // dry run qb.builder to see if it will break the grid view or not + // if so, set formula error and show empty selectQb instead + await knex(getTnPath(model, knex, tableAlias)) + .select(qb.builder) + .as('dry-run-only'); + + // if column is provided, i.e. formula has been created + if (column) { + const formula = await column.getColOptions(); + // clean the previous formula error if the formula works this time + if (formula.error) { + await FormulaColumn.update(formula.id, { + error: null, + }); + } + } + } catch (e) { + console.error(e); + if (column) { + const formula = await column.getColOptions(); + // add formula error to show in UI + await FormulaColumn.update(formula.id, { + error: e.message, + }); + // update cache to reflect the error in UI + const key = `${CacheScope.COL_FORMULA}:${column.id}`; + let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + if (o) { + o = { ...o, error: e.message }; + // set cache + await NocoCache.set(key, o); + } + } + throw new Error(`Formula error: ${e.message}`); + } +} diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts index 1fac4fbbd7..9c3a2f3033 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts @@ -28,7 +28,11 @@ export default { ) .toQuery(); } - return args.knex.raw(`CASE ${switchVal} ${query}\n END${args.colAlias}`); + return { + builder: args.knex.raw( + `CASE ${switchVal} ${query}\n END${args.colAlias}` + ), + }; }, IF: async (args: MapFnArgs) => { let query = args.knex @@ -42,62 +46,74 @@ export default { .toQuery(); if (args.pt.arguments[2]) { query += args.knex - .raw(`\n\tELSE ${(await args.fn(args.pt.arguments[2])).builder.toQuery()}`) + .raw( + `\n\tELSE ${(await args.fn(args.pt.arguments[2])).builder.toQuery()}` + ) .toQuery(); } - return args.knex.raw(`CASE ${query}\n END${args.colAlias}`); + return { builder: args.knex.raw(`CASE ${query}\n END${args.colAlias}`) }; }, - TRUE: (_args) => 1, - FALSE: (_args) => 0, + TRUE: 1, + FALSE: 0, AND: async (args: MapFnArgs) => { - return args.knex.raw( - `${args.knex - .raw( - `${( - await Promise.all( - args.pt.arguments.map(async (ar) => - (await args.fn(ar)).builder.toQuery() + return { + builder: args.knex.raw( + `${args.knex + .raw( + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar)).builder.toQuery() + ) ) - ) - ).join(' AND ')}` - ) - .wrap('(', ')') - .toQuery()}${args.colAlias}` - ); + ).join(' AND ')}` + ) + .wrap('(', ')') + .toQuery()}${args.colAlias}` + ), + }; }, OR: async (args: MapFnArgs) => { - return args.knex.raw( - `${args.knex - .raw( - `${( - await Promise.all( - args.pt.arguments.map(async (ar) => - (await args.fn(ar)).builder.toQuery() + return { + builder: args.knex.raw( + `${args.knex + .raw( + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar)).builder.toQuery() + ) ) - ) - ).join(' OR ')}` - ) - .wrap('(', ')') - .toQuery()}${args.colAlias}` - ); + ).join(' OR ')}` + ) + .wrap('(', ')') + .toQuery()}${args.colAlias}` + ), + }; }, AVG: (args: MapFnArgs) => { if (args.pt.arguments.length > 1) { - return args.fn( - { - type: 'BinaryExpression', - operator: '/', - left: { ...args.pt, callee: { name: 'SUM' } }, - right: { type: 'Literal', value: args.pt.arguments.length }, - }, - args.a, - args.prevBinaryOp - ); + return { + builder: args.fn( + { + type: 'BinaryExpression', + operator: '/', + left: { ...args.pt, callee: { name: 'SUM' } }, + right: { type: 'Literal', value: args.pt.arguments.length }, + }, + args.a, + args.prevBinaryOp + ), + }; } else { - return args.fn(args.pt.arguments[0], args.a, args.prevBinaryOp); + return { + builder: args.fn(args.pt.arguments[0], args.a, args.prevBinaryOp), + }; } }, FLOAT: async (args: MapFnArgs) => { - return (await args.fn(args.pt?.arguments?.[0])).builder.wrap('(', ')'); + return { + builder: (await args.fn(args.pt?.arguments?.[0])).builder.wrap('(', ')'), + }; }, }; diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts index b361996859..a03bc664e4 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts @@ -13,43 +13,53 @@ const pg = { POWER: 'pow', SQRT: 'sqrt', SEARCH: async (args: MapFnArgs) => { - return args.knex.raw( - `POSITION(${args.knex.raw( - (await args.fn(args.pt.arguments[1])).builder.toQuery() - )} in ${args.knex - .raw((await args.fn(args.pt.arguments[0])).builder) - .toQuery()})${args.colAlias}` - ); + return { + builder: args.knex.raw( + `POSITION(${args.knex.raw( + (await args.fn(args.pt.arguments[1])).builder.toQuery() + )} in ${args.knex + .raw((await args.fn(args.pt.arguments[0])).builder) + .toQuery()})${args.colAlias}` + ), + }; }, INT(args: MapFnArgs) { // todo: correction - return args.knex.raw( - `REGEXP_REPLACE(COALESCE(${args.fn( - args.pt.arguments[0] - )}::character varying, '0'), '[^0-9]+|\\.[0-9]+' ,'')${args.colAlias}` - ); + return { + builder: args.knex.raw( + `REGEXP_REPLACE(COALESCE(${args.fn( + args.pt.arguments[0] + )}::character varying, '0'), '[^0-9]+|\\.[0-9]+' ,'')${args.colAlias}` + ), + }; }, MID: 'SUBSTR', FLOAT: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - return knex - .raw(`CAST(${fn(pt.arguments[0])} as DOUBLE PRECISION)${colAlias}`) - .wrap('(', ')'); + return { + builder: knex + .raw(`CAST(${fn(pt.arguments[0])} as DOUBLE PRECISION)${colAlias}`) + .wrap('(', ')'), + }; }, ROUND: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - return knex.raw( - `ROUND((${fn(pt.arguments[0])})::numeric, ${ - pt?.arguments[1] ? fn(pt.arguments[1]) : 0 - }) ${colAlias}` - ); + return { + builder: knex.raw( + `ROUND((${fn(pt.arguments[0])})::numeric, ${ + pt?.arguments[1] ? fn(pt.arguments[1]) : 0 + }) ${colAlias}` + ), + }; }, DATEADD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - return knex.raw( - `${fn(pt.arguments[0])} + (${fn(pt.arguments[1])} || + return { + builder: knex.raw( + `${fn(pt.arguments[0])} + (${fn(pt.arguments[1])} || '${String(fn(pt.arguments[2])).replace( /["']/g, '' )}')::interval${colAlias}` - ); + ), + }; }, DATETIME_DIFF: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { const datetime_expr1 = fn(pt.arguments[0]); @@ -99,61 +109,75 @@ const pg = { default: sql = ''; } - return knex.raw(`${sql} ${colAlias}`); + return { builder: knex.raw(`${sql} ${colAlias}`) }; }, WEEKDAY: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { // isodow: the day of the week as Monday (1) to Sunday (7) // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday - return knex.raw( - `(EXTRACT(ISODOW FROM ${ - pt.arguments[0].type === 'Literal' - ? `date '${dayjs((await fn(pt.arguments[0])).builder).format( - 'YYYY-MM-DD' - )}'` - : fn(pt.arguments[0]) - }) - 1 - ${getWeekdayByText( - pt?.arguments[1]?.value - )} % 7 + 7) ::INTEGER % 7 ${colAlias}` - ); + return { + builder: knex.raw( + `(EXTRACT(ISODOW FROM ${ + pt.arguments[0].type === 'Literal' + ? `date '${dayjs((await fn(pt.arguments[0])).builder).format( + 'YYYY-MM-DD' + )}'` + : fn(pt.arguments[0]) + }) - 1 - ${getWeekdayByText( + pt?.arguments[1]?.value + )} % 7 + 7) ::INTEGER % 7 ${colAlias}` + ), + }; }, AND: async (args: MapFnArgs) => { - return args.knex.raw( - `CASE WHEN ${args.knex - .raw( - `${args.pt.arguments - .map(async (ar) => (await args.fn(ar, '', 'AND')).builder.toQuery()) - .join(' AND ')}` - ) - .wrap('(', ')') - .toQuery()} THEN TRUE ELSE FALSE END ${args.colAlias}` - ); + return { + builder: args.knex.raw( + `CASE WHEN ${args.knex + .raw( + `${args.pt.arguments + .map(async (ar) => + (await args.fn(ar, '', 'AND')).builder.toQuery() + ) + .join(' AND ')}` + ) + .wrap('(', ')') + .toQuery()} THEN TRUE ELSE FALSE END ${args.colAlias}` + ), + }; }, OR: async (args: MapFnArgs) => { - return args.knex.raw( - `CASE WHEN ${args.knex - .raw( - `${args.pt.arguments - .map(async (ar) => (await args.fn(ar, '', 'OR')).builder.toQuery()) - .join(' OR ')}` - ) - .wrap('(', ')') - .toQuery()} THEN TRUE ELSE FALSE END ${args.colAlias}` - ); + return { + builder: args.knex.raw( + `CASE WHEN ${args.knex + .raw( + `${args.pt.arguments + .map(async (ar) => + (await args.fn(ar, '', 'OR')).builder.toQuery() + ) + .join(' OR ')}` + ) + .wrap('(', ')') + .toQuery()} THEN TRUE ELSE FALSE END ${args.colAlias}` + ), + }; }, SUBSTR: ({ fn, knex, pt, colAlias }: MapFnArgs) => { const str = fn(pt.arguments[0]); const positionFrom = fn(pt.arguments[1] ?? 1); const numberOfCharacters = fn(pt.arguments[2] ?? ''); - return knex.raw( - `SUBSTR(${str}::TEXT, ${positionFrom}${ - numberOfCharacters ? ', ' + numberOfCharacters : '' - })${colAlias}` - ); + return { + builder: knex.raw( + `SUBSTR(${str}::TEXT, ${positionFrom}${ + numberOfCharacters ? ', ' + numberOfCharacters : '' + })${colAlias}` + ), + }; }, MOD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { const x = fn(pt.arguments[0]); const y = fn(pt.arguments[1]); - return knex.raw(`MOD((${x})::NUMERIC, (${y})::NUMERIC) ${colAlias}`); + return { + builder: knex.raw(`MOD((${x})::NUMERIC, (${y})::NUMERIC) ${colAlias}`), + }; }, }; From 88349195aadde36f253ab9be5f468f4aeb37a640 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 30 Mar 2023 23:40:49 +0530 Subject: [PATCH 03/27] refactor(nocodb): update formula functions to async and change return statement Signed-off-by: Pranav C --- .../lib/sql/functionMappings/mysql.ts | 200 +++++---- .../lib/sql/functionMappings/pg.ts | 40 +- .../lib/sql/functionMappings/sqlite.ts | 412 ++++++++++-------- 3 files changed, 349 insertions(+), 303 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts index 1c10b63434..d9a86af755 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts @@ -1,99 +1,115 @@ -// import dayjs from 'dayjs'; -// import { convertUnits } from '../helpers/convertUnits'; -// import { getWeekdayByText } from '../helpers/formulaFnHelper'; +import dayjs from 'dayjs'; +import { convertUnits } from '../helpers/convertUnits'; +import { getWeekdayByText } from '../helpers/formulaFnHelper'; import commonFns from './commonFns'; -// import type { MapFnArgs } from '../mapFunctionName'; +import type { MapFnArgs } from '../mapFunctionName'; const mysql2 = { ...commonFns, - // LEN: 'CHAR_LENGTH', - // MIN: 'LEAST', - // MAX: 'GREATEST', - // SEARCH: (args: MapFnArgs) => { - // args.pt.callee.name = 'LOCATE'; - // const temp = args.pt.arguments[0]; - // args.pt.arguments[0] = args.pt.arguments[1]; - // args.pt.arguments[1] = temp; - // }, - // INT: (args: MapFnArgs) => { - // return args.knex.raw( - // `CAST(${args.fn(args.pt.arguments[0])} as SIGNED)${args.colAlias}` - // ); - // }, - // LEFT: (args: MapFnArgs) => { - // return args.knex.raw( - // `SUBSTR(${args.fn(args.pt.arguments[0])},1,${args.fn( - // args.pt.arguments[1] - // )})${args.colAlias}` - // ); - // }, - // RIGHT: (args: MapFnArgs) => { - // return args.knex.raw( - // `SUBSTR(${args.fn(args.pt.arguments[0])}, -(${args.fn( - // args.pt.arguments[1] - // )}))${args.colAlias}` - // ); - // }, - // MID: 'SUBSTR', - // FLOAT: (args: MapFnArgs) => { - // return args.knex - // .raw( - // `CAST(CAST(${args.fn(args.pt.arguments[0])} as CHAR) AS DOUBLE)${ - // args.colAlias - // }` - // ) - // .wrap('(', ')'); - // }, - // DATEADD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - // return knex.raw( - // `CASE - // WHEN ${fn(pt.arguments[0])} LIKE '%:%' THEN - // DATE_FORMAT(DATE_ADD(${fn(pt.arguments[0])}, INTERVAL - // ${fn(pt.arguments[1])} ${String(fn(pt.arguments[2])).replace( - // /["']/g, - // '' - // )}), '%Y-%m-%d %H:%i') - // ELSE - // DATE(DATE_ADD(${fn(pt.arguments[0])}, INTERVAL - // ${fn(pt.arguments[1])} ${String(fn(pt.arguments[2])).replace( - // /["']/g, - // '' - // )})) - // END${colAlias}` - // ); - // }, - // DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - // const datetime_expr1 = fn(pt.arguments[0]); - // const datetime_expr2 = fn(pt.arguments[1]); - // - // const unit = convertUnits( - // pt.arguments[2] ? fn(pt.arguments[2]).bindings[0] : 'seconds', - // 'mysql' - // ); - // - // if (unit === 'MICROSECOND') { - // // MySQL doesn't support millisecond - // // hence change from MICROSECOND to millisecond manually - // return knex.raw( - // `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) div 1000 ${colAlias}` - // ); - // } - // return knex.raw( - // `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}` - // ); - // }, - // WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - // // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday - // return knex.raw( - // `(WEEKDAY(${ - // pt.arguments[0].type === 'Literal' - // ? `'${dayjs(fn(pt.arguments[0])).format('YYYY-MM-DD')}'` - // : fn(pt.arguments[0]) - // }) - ${getWeekdayByText( - // pt?.arguments[1]?.value - // )} % 7 + 7) % 7 ${colAlias}` - // ); - // }, + LEN: 'CHAR_LENGTH', + MIN: 'LEAST', + MAX: 'GREATEST', + SEARCH: async (args: MapFnArgs) => { + args.pt.callee.name = 'LOCATE'; + const temp = args.pt.arguments[0]; + args.pt.arguments[0] = args.pt.arguments[1]; + args.pt.arguments[1] = temp; + }, + INT: async (args: MapFnArgs) => { + return { + builder: args.knex.raw( + `CAST(${(await args.fn(args.pt.arguments[0])).builder} as SIGNED)${args.colAlias}`, + ), + }; + }, + LEFT: async (args: MapFnArgs) => { + return { + builder: args.knex.raw( + `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder},1,${(await args.fn( + args.pt.arguments[1], + )).builder})${args.colAlias}`, + ), + }; + }, + RIGHT: async (args: MapFnArgs) => { + return { + builder: args.knex.raw( + `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder}, -(${(await args.fn( + args.pt.arguments[1], + )).builder}))${args.colAlias}`, + ), + }; + }, + MID: 'SUBSTR', + FLOAT: async (args: MapFnArgs) => { + return { + builder: args.knex + .raw( + `CAST(CAST(${(await args.fn(args.pt.arguments[0])).builder} as CHAR) AS DOUBLE)${ + args.colAlias + }`, + ) + .wrap('(', ')'), + }; + }, + DATEADD: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + return { + builder: knex.raw( + `CASE + WHEN ${(await fn(pt.arguments[0])).builder} LIKE '%:%' THEN + DATE_FORMAT(DATE_ADD(${(await fn(pt.arguments[0])).builder}, INTERVAL + ${(await fn(pt.arguments[1])).builder} ${String((await fn(pt.arguments[2])).builder).replace( + /["']/g, + '', + )}), '%Y-%m-%d %H:%i') + ELSE + DATE(DATE_ADD(${(await fn(pt.arguments[0])).builder}, INTERVAL + ${(await fn(pt.arguments[1])).builder} ${String((await fn(pt.arguments[2])).builder).replace( + /["']/g, + '', + )})) + END${colAlias}`, + ), + }; + }, + DATETIME_DIFF: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + const datetime_expr1 = (await fn(pt.arguments[0])).builder; + const datetime_expr2 = (await fn(pt.arguments[1])).builder; + + const unit = convertUnits( + pt.arguments[2] ? (await fn(pt.arguments[2])).builder.bindings[0] : 'seconds', + 'mysql' + ); + + if (unit === 'MICROSECOND') { + // MySQL doesn't support millisecond + // hence change from MICROSECOND to millisecond manually + return { + builder: knex.raw( + `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) div 1000 ${colAlias}`, + ), + }; + } + return { + builder: knex.raw( + `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}`, + ), + }; + }, + WEEKDAY: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday + return { + builder: knex.raw( + `(WEEKDAY(${ + pt.arguments[0].type === 'Literal' + ? `'${dayjs((await fn(pt.arguments[0])).builder).format('YYYY-MM-DD')}'` + : (await fn(pt.arguments[0])).builder + }) - ${getWeekdayByText( + pt?.arguments[1]?.value, + )} % 7 + 7) % 7 ${colAlias}`, + ), + }; + }, }; export default mysql2; diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts index a03bc664e4..ebd1475efc 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts @@ -34,27 +34,27 @@ const pg = { }; }, MID: 'SUBSTR', - FLOAT: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + FLOAT: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { return { builder: knex - .raw(`CAST(${fn(pt.arguments[0])} as DOUBLE PRECISION)${colAlias}`) + .raw(`CAST(${(await fn(pt.arguments[0])).builder} as DOUBLE PRECISION)${colAlias}`) .wrap('(', ')'), }; }, - ROUND: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + ROUND: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { return { builder: knex.raw( - `ROUND((${fn(pt.arguments[0])})::numeric, ${ - pt?.arguments[1] ? fn(pt.arguments[1]) : 0 + `ROUND((${(await fn(pt.arguments[0])).builder})::numeric, ${ + pt?.arguments[1] ? (await fn(pt.arguments[1])).builder : 0 }) ${colAlias}` ), }; }, - DATEADD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { + DATEADD: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { return { builder: knex.raw( - `${fn(pt.arguments[0])} + (${fn(pt.arguments[1])} || - '${String(fn(pt.arguments[2])).replace( + `${(await fn(pt.arguments[0])).builder} + (${(await fn(pt.arguments[1])).builder} || + '${String((await fn(pt.arguments[2])).builder).replace( /["']/g, '' )}')::interval${colAlias}` @@ -62,8 +62,8 @@ const pg = { }; }, DATETIME_DIFF: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { - const datetime_expr1 = fn(pt.arguments[0]); - const datetime_expr2 = fn(pt.arguments[1]); + const datetime_expr1 = (await fn(pt.arguments[0])).builder; + const datetime_expr2 = (await fn(pt.arguments[1])).builder; const rawUnit = pt.arguments[2] ? (await fn(pt.arguments[2])).builder.bindings[0] : 'seconds'; @@ -121,7 +121,7 @@ const pg = { ? `date '${dayjs((await fn(pt.arguments[0])).builder).format( 'YYYY-MM-DD' )}'` - : fn(pt.arguments[0]) + : (await fn(pt.arguments[0])).builder }) - 1 - ${getWeekdayByText( pt?.arguments[1]?.value )} % 7 + 7) ::INTEGER % 7 ${colAlias}` @@ -133,10 +133,10 @@ const pg = { builder: args.knex.raw( `CASE WHEN ${args.knex .raw( - `${args.pt.arguments + `${(await Promise.all(args.pt.arguments .map(async (ar) => (await args.fn(ar, '', 'AND')).builder.toQuery() - ) + ))) .join(' AND ')}` ) .wrap('(', ')') @@ -160,10 +160,10 @@ const pg = { ), }; }, - SUBSTR: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - const str = fn(pt.arguments[0]); - const positionFrom = fn(pt.arguments[1] ?? 1); - const numberOfCharacters = fn(pt.arguments[2] ?? ''); + SUBSTR:async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + const str = (await fn(pt.arguments[0])).builder; + const positionFrom = (await fn(pt.arguments[1] ?? 1)).builder; + const numberOfCharacters = (await fn(pt.arguments[2] ?? '')).builder; return { builder: knex.raw( `SUBSTR(${str}::TEXT, ${positionFrom}${ @@ -172,9 +172,9 @@ const pg = { ), }; }, - MOD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - const x = fn(pt.arguments[0]); - const y = fn(pt.arguments[1]); + MOD: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + const x = (await fn(pt.arguments[0])).builder; + const y = (await fn(pt.arguments[1])).builder; return { builder: knex.raw(`MOD((${x})::NUMERIC, (${y})::NUMERIC) ${colAlias}`), }; diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts index 187c8cf156..bfe91ccbf5 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts @@ -1,198 +1,228 @@ -// import dayjs from 'dayjs'; -// import { convertUnits } from '../helpers/convertUnits'; -// import { getWeekdayByText } from '../helpers/formulaFnHelper'; -// import { -// convertToTargetFormat, -// getDateFormat, -// } from '../../../../../utils/dateTimeUtils'; +import dayjs from 'dayjs'; +import { convertUnits } from '../helpers/convertUnits'; +import { getWeekdayByText } from '../helpers/formulaFnHelper'; +import { + convertToTargetFormat, + getDateFormat, +} from '../../../../../utils/dateTimeUtils'; import commonFns from './commonFns'; -// import type { MapFnArgs } from '../mapFunctionName'; +import type { MapFnArgs } from '../mapFunctionName'; const sqlite3 = { ...commonFns, - // LEN: 'LENGTH', - // CEILING(args) { - // return args.knex.raw( - // `round(${args.fn(args.pt.arguments[0])} + 0.5)${args.colAlias}` - // ); - // }, - // FLOOR(args) { - // return args.knex.raw( - // `round(${args.fn(args.pt.arguments[0])} - 0.5)${args.colAlias}` - // ); - // }, - // MOD: (args: MapFnArgs) => { - // return args.fn({ - // type: 'BinaryExpression', - // operator: '%', - // left: args.pt.arguments[0], - // right: args.pt.arguments[1], - // }); - // }, - // REPEAT(args: MapFnArgs) { - // return args.knex.raw( - // `replace(printf('%.' || ${args.fn( - // args.pt.arguments[1] - // )} || 'c', '/'),'/',${args.fn(args.pt.arguments[0])})${args.colAlias}` - // ); - // }, - // NOW: 'DATE', - // SEARCH: 'INSTR', - // INT(args: MapFnArgs) { - // return args.knex.raw( - // `CAST(${args.fn(args.pt.arguments[0])} as INTEGER)${args.colAlias}` - // ); - // }, - // LEFT: (args: MapFnArgs) => { - // return args.knex.raw( - // `SUBSTR(${args.fn(args.pt.arguments[0])},1,${args.fn( - // args.pt.arguments[1] - // )})${args.colAlias}` - // ); - // }, - // RIGHT: (args: MapFnArgs) => { - // return args.knex.raw( - // `SUBSTR(${args.fn(args.pt.arguments[0])},-(${args.fn( - // args.pt.arguments[1] - // )}))${args.colAlias}` - // ); - // }, - // MID: 'SUBSTR', - // FLOAT: (args: MapFnArgs) => { - // return args.knex - // .raw(`CAST(${args.fn(args.pt.arguments[0])} as FLOAT)${args.colAlias}`) - // .wrap('(', ')'); - // }, - // DATEADD: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - // const dateIN = fn(pt.arguments[1]); - // return knex.raw( - // `CASE - // WHEN ${fn(pt.arguments[0])} LIKE '%:%' THEN - // STRFTIME('%Y-%m-%d %H:%M', DATETIME(DATETIME(${fn( - // pt.arguments[0] - // )}, 'localtime'), - // ${dateIN > 0 ? '+' : ''}${fn(pt.arguments[1])} || ' ${String( - // fn(pt.arguments[2]) - // ).replace(/["']/g, '')}')) - // ELSE - // DATE(DATETIME(${fn(pt.arguments[0])}, 'localtime'), - // ${dateIN > 0 ? '+' : ''}${fn(pt.arguments[1])} || ' ${String( - // fn(pt.arguments[2]) - // ).replace(/["']/g, '')}') - // END${colAlias}` - // ); - // }, - // DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - // let datetime_expr1 = fn(pt.arguments[0]); - // let datetime_expr2 = fn(pt.arguments[1]); - // // JULIANDAY takes YYYY-MM-DD - // if (datetime_expr1.sql === '?' && datetime_expr1.bindings?.[0]) { - // datetime_expr1 = `'${convertToTargetFormat( - // datetime_expr1.bindings[0], - // getDateFormat(datetime_expr1.bindings[0]), - // 'YYYY-MM-DD' - // )}'`; - // } - // - // if (datetime_expr2.sql === '?' && datetime_expr2.bindings?.[0]) { - // datetime_expr2 = `'${convertToTargetFormat( - // datetime_expr2.bindings[0], - // getDateFormat(datetime_expr2.bindings[0]), - // 'YYYY-MM-DD' - // )}'`; - // } - // - // const rawUnit = pt.arguments[2] - // ? fn(pt.arguments[2]).bindings[0] - // : 'seconds'; - // let sql; - // const unit = convertUnits(rawUnit, 'sqlite'); - // switch (unit) { - // case 'seconds': - // sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2}))`; - // break; - // case 'minutes': - // sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) / 60`; - // break; - // case 'hours': - // sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) / 3600`; - // break; - // case 'milliseconds': - // sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) * 1000`; - // break; - // case 'weeks': - // sql = `ROUND((JULIANDAY(${datetime_expr1}) - JULIANDAY(${datetime_expr2})) / 7)`; - // break; - // case 'months': - // sql = `(strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) * 12 + (strftime('%m', ${datetime_expr1}) - strftime('%m', ${datetime_expr2})) `; - // break; - // case 'quarters': - // sql = `(strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) * 4 + (strftime('%m', ${datetime_expr1}) - strftime('%m', ${datetime_expr2})) / 3`; - // break; - // case 'years': - // sql = `CASE - // WHEN (${datetime_expr2} < ${datetime_expr1}) THEN - // ( - // (strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) - // - (strftime('%m', ${datetime_expr1}) < strftime('%m', ${datetime_expr2}) - // OR (strftime('%m', ${datetime_expr1}) = strftime('%m', ${datetime_expr2}) - // AND strftime('%d', ${datetime_expr1}) < strftime('%d', ${datetime_expr2}))) - // ) - // WHEN (${datetime_expr2} > ${datetime_expr1}) THEN - // -1 * ( - // (strftime('%Y', ${datetime_expr2}) - strftime('%Y', ${datetime_expr1})) - // - (strftime('%m', ${datetime_expr2}) < strftime('%m', ${datetime_expr1}) - // OR (strftime('%m', ${datetime_expr2}) = strftime('%m', ${datetime_expr1}) - // AND strftime('%d', ${datetime_expr2}) < strftime('%d', ${datetime_expr1}))) - // ) - // ELSE 0 - // END`; - // break; - // case 'days': - // sql = `JULIANDAY(${datetime_expr1}) - JULIANDAY(${datetime_expr2})`; - // break; - // default: - // sql = ''; - // } - // return knex.raw(`${sql} ${colAlias}`); - // }, - // WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => { - // // strftime('%w', date) - day of week 0 - 6 with Sunday == 0 - // // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday - // return knex.raw( - // `(strftime('%w', ${ - // pt.arguments[0].type === 'Literal' - // ? `'${dayjs(fn(pt.arguments[0])).format('YYYY-MM-DD')}'` - // : fn(pt.arguments[0]) - // }) - 1 - ${getWeekdayByText( - // pt?.arguments[1]?.value - // )} % 7 + 7) % 7 ${colAlias}` - // ); - // }, - // AND: (args: MapFnArgs) => { - // return args.knex.raw( - // `CASE WHEN ${args.knex - // .raw( - // `${args.pt.arguments - // .map((ar) => args.fn(ar, '', 'AND').toQuery()) - // .join(' AND ')}` - // ) - // .wrap('(', ')') - // .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` - // ); - // }, - // OR: (args: MapFnArgs) => { - // return args.knex.raw( - // `CASE WHEN ${args.knex - // .raw( - // `${args.pt.arguments - // .map((ar) => args.fn(ar, '', 'OR').toQuery()) - // .join(' OR ')}` - // ) - // .wrap('(', ')') - // .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` - // ); - // }, + LEN: 'LENGTH', + async CEILING(args) { + return args.knex.raw( + `round(${(await args.fn(args.pt.arguments[0])).builder} + 0.5)${ + args.colAlias + }` + ); + }, + async FLOOR(args) { + return args.knex.raw( + `round(${(await args.fn(args.pt.arguments[0])).builder} - 0.5)${ + args.colAlias + }` + ); + }, + MOD: async (args: MapFnArgs) => { + return ( + await args.fn({ + type: 'BinaryExpression', + operator: '%', + left: args.pt.arguments[0], + right: args.pt.arguments[1], + }) + ).builder; + }, + async REPEAT(args: MapFnArgs) { + return args.knex.raw( + `replace(printf('%.' || ${ + (await args.fn(args.pt.arguments[1])).builder + } || 'c', '/'),'/',${(await args.fn(args.pt.arguments[0])).builder})${ + args.colAlias + }` + ); + }, + NOW: 'DATE', + SEARCH: 'INSTR', + async INT(args: MapFnArgs) { + return args.knex.raw( + `CAST(${(await args.fn(args.pt.arguments[0])).builder} as INTEGER)${ + args.colAlias + }` + ); + }, + LEFT: async (args: MapFnArgs) => { + return args.knex.raw( + `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder},1,${ + (await args.fn(args.pt.arguments[1])).builder + })${args.colAlias}` + ); + }, + RIGHT: async (args: MapFnArgs) => { + return args.knex.raw( + `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder},-(${ + (await args.fn(args.pt.arguments[1])).builder + }))${args.colAlias}` + ); + }, + MID: 'SUBSTR', + FLOAT: async (args: MapFnArgs) => { + return args.knex + .raw( + `CAST(${(await args.fn(args.pt.arguments[0])).builder} as FLOAT)${ + args.colAlias + }` + ) + .wrap('(', ')'); + }, + DATEADD: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + const dateIN = (await fn(pt.arguments[1])).builder; + return knex.raw( + `CASE + WHEN ${(await fn(pt.arguments[0])).builder} LIKE '%:%' THEN + STRFTIME('%Y-%m-%d %H:%M', DATETIME(DATETIME(${fn( + pt.arguments[0] + )}, 'localtime'), + ${dateIN > 0 ? '+' : ''}${ + (await fn(pt.arguments[1])).builder + } || ' ${String((await fn(pt.arguments[2])).builder).replace( + /["']/g, + '' + )}')) + ELSE + DATE(DATETIME(${(await fn(pt.arguments[0])).builder}, 'localtime'), + ${dateIN > 0 ? '+' : ''}${ + (await fn(pt.arguments[1])).builder + } || ' ${String((await fn(pt.arguments[2])).builder).replace( + /["']/g, + '' + )}') + END${colAlias}` + ); + }, + DATETIME_DIFF: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + let datetime_expr1 = (await fn(pt.arguments[0])).builder; + let datetime_expr2 = (await fn(pt.arguments[1])).builder; + // JULIANDAY takes YYYY-MM-DD + if (datetime_expr1.sql === '?' && datetime_expr1.bindings?.[0]) { + datetime_expr1 = `'${convertToTargetFormat( + datetime_expr1.bindings[0], + getDateFormat(datetime_expr1.bindings[0]), + 'YYYY-MM-DD' + )}'`; + } + + if (datetime_expr2.sql === '?' && datetime_expr2.bindings?.[0]) { + datetime_expr2 = `'${convertToTargetFormat( + datetime_expr2.bindings[0], + getDateFormat(datetime_expr2.bindings[0]), + 'YYYY-MM-DD' + )}'`; + } + + const rawUnit = pt.arguments[2] + ? (await fn(pt.arguments[2])).builder.bindings[0] + : 'seconds'; + let sql; + const unit = convertUnits(rawUnit, 'sqlite'); + switch (unit) { + case 'seconds': + sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2}))`; + break; + case 'minutes': + sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) / 60`; + break; + case 'hours': + sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) / 3600`; + break; + case 'milliseconds': + sql = `(strftime('%s', ${datetime_expr1}) - strftime('%s', ${datetime_expr2})) * 1000`; + break; + case 'weeks': + sql = `ROUND((JULIANDAY(${datetime_expr1}) - JULIANDAY(${datetime_expr2})) / 7)`; + break; + case 'months': + sql = `(strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) * 12 + (strftime('%m', ${datetime_expr1}) - strftime('%m', ${datetime_expr2})) `; + break; + case 'quarters': + sql = `(strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) * 4 + (strftime('%m', ${datetime_expr1}) - strftime('%m', ${datetime_expr2})) / 3`; + break; + case 'years': + sql = `CASE + WHEN (${datetime_expr2} < ${datetime_expr1}) THEN + ( + (strftime('%Y', ${datetime_expr1}) - strftime('%Y', ${datetime_expr2})) + - (strftime('%m', ${datetime_expr1}) < strftime('%m', ${datetime_expr2}) + OR (strftime('%m', ${datetime_expr1}) = strftime('%m', ${datetime_expr2}) + AND strftime('%d', ${datetime_expr1}) < strftime('%d', ${datetime_expr2}))) + ) + WHEN (${datetime_expr2} > ${datetime_expr1}) THEN + -1 * ( + (strftime('%Y', ${datetime_expr2}) - strftime('%Y', ${datetime_expr1})) + - (strftime('%m', ${datetime_expr2}) < strftime('%m', ${datetime_expr1}) + OR (strftime('%m', ${datetime_expr2}) = strftime('%m', ${datetime_expr1}) + AND strftime('%d', ${datetime_expr2}) < strftime('%d', ${datetime_expr1}))) + ) + ELSE 0 + END`; + break; + case 'days': + sql = `JULIANDAY(${datetime_expr1}) - JULIANDAY(${datetime_expr2})`; + break; + default: + sql = ''; + } + return knex.raw(`${sql} ${colAlias}`); + }, + WEEKDAY: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + // strftime('%w', date) - day of week 0 - 6 with Sunday == 0 + // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday + return knex.raw( + `(strftime('%w', ${ + pt.arguments[0].type === 'Literal' + ? `'${dayjs((await fn(pt.arguments[0])).builder).format( + 'YYYY-MM-DD' + )}'` + : (await fn(pt.arguments[0])).builder + }) - 1 - ${getWeekdayByText( + pt?.arguments[1]?.value + )} % 7 + 7) % 7 ${colAlias}` + ); + }, + AND: async (args: MapFnArgs) => { + return args.knex.raw( + `CASE WHEN ${args.knex + .raw( + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar, '', 'AND')).builder.toQuery() + ) + ) + ).join(' AND ')}` + ) + .wrap('(', ')') + .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` + ); + }, + OR: async (args: MapFnArgs) => { + return args.knex.raw( + `CASE WHEN ${args.knex + .raw( + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar, '', 'OR')).builder.toQuery() + ) + ) + ).join(' OR ')}` + ) + .wrap('(', ')') + .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` + ); + }, }; export default sqlite3; From 15179c97602d05df4f223db47ff0203d6a4396d7 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 31 Mar 2023 11:37:48 +0530 Subject: [PATCH 04/27] refactor(nocodb): reuse the column builder reference across all the formula Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index c4b1f6a1ee..9f78eaa481 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -1283,7 +1283,8 @@ class BaseModelSqlv2 { private async getSelectQueryBuilderForFormula( column: Column, tableAlias?: string, - validateFormula = false + validateFormula = false, + aliasToColumnBuilder = {} ) { const formula = await column.getColOptions(); if (formula.error) throw new Error(`Formula error: ${formula.error}`); @@ -1293,7 +1294,7 @@ class BaseModelSqlv2 { this.dbDriver, this.model, column, - {}, + aliasToColumnBuilder, tableAlias, validateFormula ); @@ -1517,7 +1518,7 @@ class BaseModelSqlv2 { validateFormula, }: { fieldsSet?: Set; - qb: Knex.QueryBuilder; + qb: Knex.QueryBuilder & Knex.QueryInterface; columns?: Column[]; fields?: string[] | string; extractPkAndPv?: boolean; @@ -1525,6 +1526,8 @@ class BaseModelSqlv2 { alias?: string; validateFormula?: boolean; }): Promise { + // keep a common object for all columns to share across all columns + const aliasToColumnBuilder = {}; let viewOrTableColumns: Column[] | { fk_column_id?: string }[]; const res = {}; @@ -1588,7 +1591,8 @@ class BaseModelSqlv2 { const selectQb = await this.getSelectQueryBuilderForFormula( qrValueColumn, alias, - validateFormula + validateFormula, + aliasToColumnBuilder ); qb.select({ [column.column_name]: selectQb.builder, @@ -1622,7 +1626,8 @@ class BaseModelSqlv2 { const selectQb = await this.getSelectQueryBuilderForFormula( barcodeValueColumn, alias, - validateFormula + validateFormula, + aliasToColumnBuilder ); qb.select({ [column.column_name]: selectQb.builder, @@ -1647,7 +1652,8 @@ class BaseModelSqlv2 { const selectQb = await this.getSelectQueryBuilderForFormula( column, alias, - validateFormula + validateFormula, + aliasToColumnBuilder ); qb.select( this.dbDriver.raw(`?? as ??`, [ From 5b32ec7a5db4b2879e5194196c06618be0095e3a Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 31 Mar 2023 14:19:29 +0530 Subject: [PATCH 05/27] fix(nocodb): handle function handler which is not returning a builder Signed-off-by: Pranav C --- .../sql/formulav2/formulaQueryBuilderv2.ts | 97 ++++--------------- 1 file changed, 20 insertions(+), 77 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index a3910464ef..1aa1dd12c4 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -613,7 +613,7 @@ async function _formulaQueryBuilder( break; default: { - const res = mapFunctionName({ + const res = await mapFunctionName({ pt, knex, alias, @@ -630,23 +630,25 @@ async function _formulaQueryBuilder( return { builder: knex.raw( - `${pt.callee.name}(${pt.arguments - .map((arg) => { - const query = fn(arg).toQuery(); - if (pt.callee.name === 'CONCAT') { - if (knex.clientType() === 'mysql2') { - // mysql2: CONCAT() returns NULL if any argument is NULL. - // adding IFNULL to convert NULL values to empty strings - return `IFNULL(${query}, '')`; - } else { - // do nothing - // pg / mssql: Concatenate all arguments. NULL arguments are ignored. - // sqlite3: special handling - See BinaryExpression + `${pt.callee.name}(${( + await Promise.all( + pt.arguments.map(async (arg) => { + const query = (await fn(arg)).builder.toQuery(); + if (pt.callee.name === 'CONCAT') { + if (knex.clientType() === 'mysql2') { + // mysql2: CONCAT() returns NULL if any argument is NULL. + // adding IFNULL to convert NULL values to empty strings + return `IFNULL(${query}, '')`; + } else { + // do nothing + // pg / mssql: Concatenate all arguments. NULL arguments are ignored. + // sqlite3: special handling - See BinaryExpression + } } - } - return query; - }) - .join()})${colAlias}`.replace(/\?/g, '\\?') + return query; + }) + ) + ).join()})${colAlias}`.replace(/\?/g, '\\?') ), }; } else if (pt.type === 'Literal') { @@ -844,7 +846,7 @@ export default async function formulaQueryBuilderv2( // dry run qb.builder to see if it will break the grid view or not // if so, set formula error and show empty selectQb instead await knex(getTnPath(model, knex, tableAlias)) - .select(qb.builder) + .select(knex.raw(`?? as ??`, [qb.builder, '__dry_run_alias'])) .as('dry-run-only'); // if column is provided, i.e. formula has been created @@ -878,62 +880,3 @@ export default async function formulaQueryBuilderv2( } return qb; } - -export async function validateFormula( - _tree, - alias, - knex: XKnex, - model: Model, - column?: Column, - aliasToColumn = {}, - tableAlias?: string -) { - // register jsep curly hook once only - jsep.plugins.register(jsepCurlyHook); - // generate qb - const qb = await _formulaQueryBuilder( - _tree, - alias, - knex, - model, - aliasToColumn, - tableAlias - ); - - try { - // dry run qb.builder to see if it will break the grid view or not - // if so, set formula error and show empty selectQb instead - await knex(getTnPath(model, knex, tableAlias)) - .select(qb.builder) - .as('dry-run-only'); - - // if column is provided, i.e. formula has been created - if (column) { - const formula = await column.getColOptions(); - // clean the previous formula error if the formula works this time - if (formula.error) { - await FormulaColumn.update(formula.id, { - error: null, - }); - } - } - } catch (e) { - console.error(e); - if (column) { - const formula = await column.getColOptions(); - // add formula error to show in UI - await FormulaColumn.update(formula.id, { - error: e.message, - }); - // update cache to reflect the error in UI - const key = `${CacheScope.COL_FORMULA}:${column.id}`; - let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); - if (o) { - o = { ...o, error: e.message }; - // set cache - await NocoCache.set(key, o); - } - } - throw new Error(`Formula error: ${e.message}`); - } -} From 5f2419e8bf0959d80ba48f140758d3a6618e33e0 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Fri, 31 Mar 2023 15:32:23 +0530 Subject: [PATCH 06/27] test: mega table generator, disabled by default Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- tests/playwright/tests/megaTable.spec.ts | 123 +++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 tests/playwright/tests/megaTable.spec.ts diff --git a/tests/playwright/tests/megaTable.spec.ts b/tests/playwright/tests/megaTable.spec.ts new file mode 100644 index 0000000000..456b16c36f --- /dev/null +++ b/tests/playwright/tests/megaTable.spec.ts @@ -0,0 +1,123 @@ +import { test } from '@playwright/test'; +import setup from '../setup'; +import { UITypes } from 'nocodb-sdk'; +import { Api } from 'nocodb-sdk'; +let api: Api; + +// configuration + +// To use, modify the test.skip to test.only +// Add columns as required to megaTblColumns +// Add row count as required to megaTblRows + +const megaTblColumns = [ + { type: 'SingleLineText', count: 3 }, + { type: 'Number', count: 3 }, + { type: 'Checkbox', count: 3 }, + { type: 'SingleSelect', count: 3 }, + { type: 'MultiSelect', count: 3 }, + { type: 'Formula', count: 0 }, +]; +const megaTblRows = 50000; +const bulkInsertAfterRows = 1000; + +test.describe.serial('Test table', () => { + let context: any; + + test.beforeEach(async ({ page }) => { + context = await setup({ page, isEmptyProject: true }); + + api = new Api({ + baseURL: `http://localhost:8080/`, + headers: { + 'xc-auth': context.token, + }, + }); + }); + + test.skip('mega table', async ({ page }) => { + let table_1; + const table_1_columns = []; + + // a Primary key column & display column + table_1_columns.push( + { + column_name: 'Id', + title: 'Id', + uidt: UITypes.ID, + }, + { + column_name: 'SingleLineText', + title: 'SingleLineText', + uidt: UITypes.SingleLineText, + pv: true, + } + ); + + for (let i = 0; i < megaTblColumns.length; i++) { + for (let j = 0; j < megaTblColumns[i].count; j++) { + const column = { + column_name: `${megaTblColumns[i].type}${j}`, + title: `${megaTblColumns[i].type}${j}`, + uidt: UITypes[megaTblColumns[i].type], + }; + if (megaTblColumns[i].type === 'SingleSelect' || megaTblColumns[i].type === 'MultiSelect') { + column['dtxp'] = "'jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec'"; + } else if (megaTblColumns[i].type === 'Formula') { + column['formula_raw'] = '{Id}'; + } + table_1_columns.push(column); + } + } + + try { + const project = await api.project.read(context.project.id); + table_1 = await api.base.tableCreate(context.project.id, project.bases?.[0].id, { + table_name: 'table_1', + title: 'table_1', + columns: table_1_columns, + }); + + const table_1_rows = []; + for (let rowCnt = 0; rowCnt < megaTblRows; rowCnt++) { + const row = { + Id: rowCnt + 1, + SingleLineText: `SingleLineText${rowCnt + 1}`, + }; + for (let colCnt = 0; colCnt < megaTblColumns.length; colCnt++) { + for (let colInstanceCnt = 0; colInstanceCnt < megaTblColumns[colCnt].count; colInstanceCnt++) { + const columnName = `${megaTblColumns[colCnt].type}${colInstanceCnt}`; + if (megaTblColumns[colCnt].type === 'SingleLineText') { + row[columnName] = `SingleLineText${rowCnt + 1}`; + } else if (megaTblColumns[colCnt].type === 'Number') { + row[columnName] = rowCnt + 1; + } else if (megaTblColumns[colCnt].type === 'Checkbox') { + row[columnName] = rowCnt % 2 === 0; + } else if (megaTblColumns[colCnt].type === 'SingleSelect') { + row[columnName] = 'jan'; + } else if (megaTblColumns[colCnt].type === 'MultiSelect') { + row[columnName] = 'jan,feb,mar,apr'; + } + } + } + table_1_rows.push(row); + + // insert as soon as we have 1k records ready + if (table_1_rows.length === bulkInsertAfterRows) { + await api.dbTableRow.bulkCreate('noco', context.project.id, table_1.id, table_1_rows); + console.log(`table_1_rows ${rowCnt + 1} created`); + table_1_rows.length = 0; + } + } + + if (table_1_rows.length > 0) { + await api.dbTableRow.bulkCreate('noco', context.project.id, table_1.id, table_1_rows); + console.log(`table_1_rows ${megaTblRows} created`); + } + } catch (e) { + console.log(e); + } + + await page.reload(); + }); +}); From 5670f510193c9d4878c0a508c5f07033207bf0a1 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 31 Mar 2023 15:44:14 +0530 Subject: [PATCH 07/27] fix(nocodb): AVG function handler return value correction Signed-off-by: Pranav C --- .../lib/sql/functionMappings/commonFns.ts | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts index 9c3a2f3033..22eb5ce1f0 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/commonFns.ts @@ -91,24 +91,20 @@ export default { ), }; }, - AVG: (args: MapFnArgs) => { + AVG: async (args: MapFnArgs) => { if (args.pt.arguments.length > 1) { - return { - builder: args.fn( - { - type: 'BinaryExpression', - operator: '/', - left: { ...args.pt, callee: { name: 'SUM' } }, - right: { type: 'Literal', value: args.pt.arguments.length }, - }, - args.a, - args.prevBinaryOp - ), - }; + return args.fn( + { + type: 'BinaryExpression', + operator: '/', + left: { ...args.pt, callee: { name: 'SUM' } }, + right: { type: 'Literal', value: args.pt.arguments.length }, + }, + args.a, + args.prevBinaryOp + ); } else { - return { - builder: args.fn(args.pt.arguments[0], args.a, args.prevBinaryOp), - }; + return args.fn(args.pt.arguments[0], args.a, args.prevBinaryOp); } }, FLOAT: async (args: MapFnArgs) => { From 2539aba5d95dec2df6e57692932b4352ac08c8f7 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 31 Mar 2023 15:49:27 +0530 Subject: [PATCH 08/27] chore(nocodb): lint Signed-off-by: Pranav C --- .../lib/sql/functionMappings/mysql.ts | 54 ++++++++++--------- .../lib/sql/functionMappings/pg.ts | 24 ++++++--- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts index d9a86af755..7844af1d05 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts @@ -18,25 +18,27 @@ const mysql2 = { INT: async (args: MapFnArgs) => { return { builder: args.knex.raw( - `CAST(${(await args.fn(args.pt.arguments[0])).builder} as SIGNED)${args.colAlias}`, + `CAST(${(await args.fn(args.pt.arguments[0])).builder} as SIGNED)${ + args.colAlias + }` ), }; }, LEFT: async (args: MapFnArgs) => { return { builder: args.knex.raw( - `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder},1,${(await args.fn( - args.pt.arguments[1], - )).builder})${args.colAlias}`, + `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder},1,${ + (await args.fn(args.pt.arguments[1])).builder + })${args.colAlias}` ), }; }, RIGHT: async (args: MapFnArgs) => { return { builder: args.knex.raw( - `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder}, -(${(await args.fn( - args.pt.arguments[1], - )).builder}))${args.colAlias}`, + `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder}, -(${ + (await args.fn(args.pt.arguments[1])).builder + }))${args.colAlias}` ), }; }, @@ -45,9 +47,9 @@ const mysql2 = { return { builder: args.knex .raw( - `CAST(CAST(${(await args.fn(args.pt.arguments[0])).builder} as CHAR) AS DOUBLE)${ - args.colAlias - }`, + `CAST(CAST(${ + (await args.fn(args.pt.arguments[0])).builder + } as CHAR) AS DOUBLE)${args.colAlias}` ) .wrap('(', ')'), }; @@ -58,17 +60,15 @@ const mysql2 = { `CASE WHEN ${(await fn(pt.arguments[0])).builder} LIKE '%:%' THEN DATE_FORMAT(DATE_ADD(${(await fn(pt.arguments[0])).builder}, INTERVAL - ${(await fn(pt.arguments[1])).builder} ${String((await fn(pt.arguments[2])).builder).replace( - /["']/g, - '', - )}), '%Y-%m-%d %H:%i') + ${(await fn(pt.arguments[1])).builder} ${String( + (await fn(pt.arguments[2])).builder + ).replace(/["']/g, '')}), '%Y-%m-%d %H:%i') ELSE DATE(DATE_ADD(${(await fn(pt.arguments[0])).builder}, INTERVAL - ${(await fn(pt.arguments[1])).builder} ${String((await fn(pt.arguments[2])).builder).replace( - /["']/g, - '', - )})) - END${colAlias}`, + ${(await fn(pt.arguments[1])).builder} ${String( + (await fn(pt.arguments[2])).builder + ).replace(/["']/g, '')})) + END${colAlias}` ), }; }, @@ -77,7 +77,9 @@ const mysql2 = { const datetime_expr2 = (await fn(pt.arguments[1])).builder; const unit = convertUnits( - pt.arguments[2] ? (await fn(pt.arguments[2])).builder.bindings[0] : 'seconds', + pt.arguments[2] + ? (await fn(pt.arguments[2])).builder.bindings[0] + : 'seconds', 'mysql' ); @@ -86,13 +88,13 @@ const mysql2 = { // hence change from MICROSECOND to millisecond manually return { builder: knex.raw( - `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) div 1000 ${colAlias}`, + `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) div 1000 ${colAlias}` ), }; } return { builder: knex.raw( - `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}`, + `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}` ), }; }, @@ -102,11 +104,13 @@ const mysql2 = { builder: knex.raw( `(WEEKDAY(${ pt.arguments[0].type === 'Literal' - ? `'${dayjs((await fn(pt.arguments[0])).builder).format('YYYY-MM-DD')}'` + ? `'${dayjs((await fn(pt.arguments[0])).builder).format( + 'YYYY-MM-DD' + )}'` : (await fn(pt.arguments[0])).builder }) - ${getWeekdayByText( - pt?.arguments[1]?.value, - )} % 7 + 7) % 7 ${colAlias}`, + pt?.arguments[1]?.value + )} % 7 + 7) % 7 ${colAlias}` ), }; }, diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts index ebd1475efc..222b0751bc 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts @@ -37,7 +37,11 @@ const pg = { FLOAT: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { return { builder: knex - .raw(`CAST(${(await fn(pt.arguments[0])).builder} as DOUBLE PRECISION)${colAlias}`) + .raw( + `CAST(${ + (await fn(pt.arguments[0])).builder + } as DOUBLE PRECISION)${colAlias}` + ) .wrap('(', ')'), }; }, @@ -53,7 +57,9 @@ const pg = { DATEADD: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { return { builder: knex.raw( - `${(await fn(pt.arguments[0])).builder} + (${(await fn(pt.arguments[1])).builder} || + `${(await fn(pt.arguments[0])).builder} + (${ + (await fn(pt.arguments[1])).builder + } || '${String((await fn(pt.arguments[2])).builder).replace( /["']/g, '' @@ -133,11 +139,13 @@ const pg = { builder: args.knex.raw( `CASE WHEN ${args.knex .raw( - `${(await Promise.all(args.pt.arguments - .map(async (ar) => - (await args.fn(ar, '', 'AND')).builder.toQuery() - ))) - .join(' AND ')}` + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar, '', 'AND')).builder.toQuery() + ) + ) + ).join(' AND ')}` ) .wrap('(', ')') .toQuery()} THEN TRUE ELSE FALSE END ${args.colAlias}` @@ -160,7 +168,7 @@ const pg = { ), }; }, - SUBSTR:async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + SUBSTR: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { const str = (await fn(pt.arguments[0])).builder; const positionFrom = (await fn(pt.arguments[1] ?? 1)).builder; const numberOfCharacters = (await fn(pt.arguments[2] ?? '')).builder; From 7cac9202743a0a99c594bb5e67cb218c4ccda72d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 31 Mar 2023 16:47:27 +0530 Subject: [PATCH 09/27] fix(nocodb): formula function builder return statement correction Signed-off-by: Pranav C --- .../lib/sql/functionMappings/mssql.ts | 182 ++++++++------- .../lib/sql/functionMappings/sqlite.ts | 216 ++++++++++-------- 2 files changed, 221 insertions(+), 177 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts index 6bf5e41831..ad38851993 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts @@ -14,9 +14,7 @@ const mssql = { for (const [i, arg] of Object.entries(args.pt.arguments)) { if (+i === args.pt.arguments.length - 1) { query += args.knex - .raw( - `\n\tElse ${(await await args.fn(arg)).builder.builder.toQuery()}` - ) + .raw(`\n\tElse ${(await args.fn(arg)).builder.builder.toQuery()}`) .toQuery(); } else { query += args.knex @@ -37,7 +35,7 @@ const mssql = { .toQuery(); } } - return args.knex.raw(`Case ${query}\n End${args.colAlias}`); + return { builder: args.knex.raw(`Case ${query}\n End${args.colAlias}`) }; }, MAX: async (args: MapFnArgs) => { if (args.pt.arguments.length === 1) { @@ -66,18 +64,20 @@ const mssql = { } } - return args.knex.raw(`Case ${query}\n End${args.colAlias}`); + return { builder: args.knex.raw(`Case ${query}\n End${args.colAlias}`) }; }, LOG: async (args: MapFnArgs) => { - return args.knex.raw( - `LOG(${( - await Promise.all( - args.pt.arguments - .reverse() - .map(async (ar) => (await args.fn(ar)).builder.toQuery()) - ) - ).join(',')})${args.colAlias}` - ); + return { + builder: args.knex.raw( + `LOG(${( + await Promise.all( + args.pt.arguments + .reverse() + .map(async (ar) => (await args.fn(ar)).builder.toQuery()) + ) + ).join(',')})${args.colAlias}` + ), + }; }, MOD: (pt) => { Object.assign(pt, { @@ -96,94 +96,118 @@ const mssql = { args.pt.arguments[1] = temp; }, INT: async (args: MapFnArgs) => { - return args.knex.raw( - `CASE WHEN ISNUMERIC(${( - await args.fn(args.pt.arguments[0]) - ).builder.toQuery()}) = 1 THEN FLOOR(${( - await args.fn(args.pt.arguments[0]) - ).builder.toQuery()}) ELSE 0 END${args.colAlias}` - ); + return { + builder: args.knex.raw( + `CASE WHEN ISNUMERIC(${( + await args.fn(args.pt.arguments[0]) + ).builder.toQuery()}) = 1 THEN FLOOR(${( + await args.fn(args.pt.arguments[0]) + ).builder.toQuery()}) ELSE 0 END${args.colAlias}` + ), + }; }, MID: 'SUBSTR', - FLOAT: (args: MapFnArgs) => { - return args.knex - .raw(`CAST(${args.fn(args.pt.arguments[0])} as FLOAT)${args.colAlias}`) - .wrap('(', ')'); + FLOAT: async (args: MapFnArgs) => { + return { + builder: args.knex + .raw( + `CAST(${(await args.fn(args.pt.arguments[0])).builder} as FLOAT)${ + args.colAlias + }` + ) + .wrap('(', ')'), + }; }, DATEADD: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { const dateIN = (await fn(pt.arguments[1])).builder; - return knex.raw( - `CASE - WHEN ${fn(pt.arguments[0])} LIKE '%:%' THEN - FORMAT(DATEADD(${String(fn(pt.arguments[2])).replace(/["']/g, '')}, - ${dateIN > 0 ? '+' : ''}${fn(pt.arguments[1])}, ${fn( - pt.arguments[0] - )}), 'yyyy-MM-dd HH:mm') + return { + builder: knex.raw( + `CASE + WHEN ${(await fn(pt.arguments[0])).builder} LIKE '%:%' THEN + FORMAT(DATEADD(${String((await fn(pt.arguments[2])).builder).replace( + /["']/g, + '' + )}, + ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${ + (await fn(pt.arguments[0])).builder + }), 'yyyy-MM-dd HH:mm') ELSE - FORMAT(DATEADD(${String(fn(pt.arguments[2])).replace(/["']/g, '')}, - ${dateIN > 0 ? '+' : ''}${fn(pt.arguments[1])}, ${fn( - pt.arguments[0] - )}), 'yyyy-MM-dd') + FORMAT(DATEADD(${String((await fn(pt.arguments[2])).builder).replace( + /["']/g, + '' + )}, + ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${fn( + pt.arguments[0] + )}), 'yyyy-MM-dd') END${colAlias}` - ); + ), + }; }, DATETIME_DIFF: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { - const datetime_expr1 = fn(pt.arguments[0]); - const datetime_expr2 = fn(pt.arguments[1]); + const datetime_expr1 = (await fn(pt.arguments[0])).builder; + const datetime_expr2 = (await fn(pt.arguments[1])).builder; const rawUnit = pt.arguments[2] ? (await fn(pt.arguments[2])).builder.bindings[0] : 'seconds'; const unit = convertUnits(rawUnit, 'mssql'); - return knex.raw( - `DATEDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}` - ); + return { + builder: knex.raw( + `DATEDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}` + ), + }; }, WEEKDAY: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { // DATEPART(WEEKDAY, DATE): sunday = 1, monday = 2, ..., saturday = 7 // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday - return knex.raw( - `(DATEPART(WEEKDAY, ${ - pt.arguments[0].type === 'Literal' - ? `'${dayjs((await fn(pt.arguments[0])).builder).format( - 'YYYY-MM-DD' - )}'` - : fn(pt.arguments[0]) - }) - 2 - ${getWeekdayByText( - pt?.arguments[1]?.value - )} % 7 + 7) % 7 ${colAlias}` - ); + return { + builder: knex.raw( + `(DATEPART(WEEKDAY, ${ + pt.arguments[0].type === 'Literal' + ? `'${dayjs((await fn(pt.arguments[0])).builder).format( + 'YYYY-MM-DD' + )}'` + : fn(pt.arguments[0]) + }) - 2 - ${getWeekdayByText( + pt?.arguments[1]?.value + )} % 7 + 7) % 7 ${colAlias}` + ), + }; }, AND: async (args: MapFnArgs) => { - return args.knex.raw( - `CASE WHEN ${args.knex - .raw( - `${( - await Promise.all( - args.pt.arguments.map(async (ar) => - (await args.fn(ar, '', 'AND')).builder.toQuery() + return { + builder: args.knex.raw( + `CASE WHEN ${args.knex + .raw( + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar, '', 'AND')).builder.toQuery() + ) ) - ) - ).join(' AND ')}` - ) - .wrap('(', ')') - .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` - ); + ).join(' AND ')}` + ) + .wrap('(', ')') + .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` + ), + }; }, OR: async (args: MapFnArgs) => { - return args.knex.raw( - `CASE WHEN ${args.knex - .raw( - `${( - await Promise.all( - args.pt.arguments.map(async (ar) => - (await args.fn(ar, '', 'OR')).builder.toQuery() + return { + builder: args.knex.raw( + `CASE WHEN ${args.knex + .raw( + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar, '', 'OR')).builder.toQuery() + ) ) - ) - ).join(' OR ')}` - ) - .wrap('(', ')') - .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` - ); + ).join(' OR ')}` + ) + .wrap('(', ')') + .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` + ), + }; }, }; diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts index bfe91ccbf5..f3c219efe5 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts @@ -12,95 +12,109 @@ const sqlite3 = { ...commonFns, LEN: 'LENGTH', async CEILING(args) { - return args.knex.raw( - `round(${(await args.fn(args.pt.arguments[0])).builder} + 0.5)${ - args.colAlias - }` - ); + return { + builder: args.knex.raw( + `round(${(await args.fn(args.pt.arguments[0])).builder} + 0.5)${ + args.colAlias + }` + ), + }; }, async FLOOR(args) { - return args.knex.raw( - `round(${(await args.fn(args.pt.arguments[0])).builder} - 0.5)${ - args.colAlias - }` - ); + return { + builder: args.knex.raw( + `round(${(await args.fn(args.pt.arguments[0])).builder} - 0.5)${ + args.colAlias + }` + ), + }; }, MOD: async (args: MapFnArgs) => { - return ( - await args.fn({ - type: 'BinaryExpression', - operator: '%', - left: args.pt.arguments[0], - right: args.pt.arguments[1], - }) - ).builder; + return args.fn({ + type: 'BinaryExpression', + operator: '%', + left: args.pt.arguments[0], + right: args.pt.arguments[1], + }); }, async REPEAT(args: MapFnArgs) { - return args.knex.raw( - `replace(printf('%.' || ${ - (await args.fn(args.pt.arguments[1])).builder - } || 'c', '/'),'/',${(await args.fn(args.pt.arguments[0])).builder})${ - args.colAlias - }` - ); + return { + builder: args.knex.raw( + `replace(printf('%.' || ${ + (await args.fn(args.pt.arguments[1])).builder + } || 'c', '/'),'/',${(await args.fn(args.pt.arguments[0])).builder})${ + args.colAlias + }` + ), + }; }, NOW: 'DATE', SEARCH: 'INSTR', async INT(args: MapFnArgs) { - return args.knex.raw( - `CAST(${(await args.fn(args.pt.arguments[0])).builder} as INTEGER)${ - args.colAlias - }` - ); + return { + builder: args.knex.raw( + `CAST(${(await args.fn(args.pt.arguments[0])).builder} as INTEGER)${ + args.colAlias + }` + ), + }; }, LEFT: async (args: MapFnArgs) => { - return args.knex.raw( - `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder},1,${ - (await args.fn(args.pt.arguments[1])).builder - })${args.colAlias}` - ); + return { + builder: args.knex.raw( + `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder},1,${ + (await args.fn(args.pt.arguments[1])).builder + })${args.colAlias}` + ), + }; }, RIGHT: async (args: MapFnArgs) => { - return args.knex.raw( - `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder},-(${ - (await args.fn(args.pt.arguments[1])).builder - }))${args.colAlias}` - ); + return { + builder: args.knex.raw( + `SUBSTR(${(await args.fn(args.pt.arguments[0])).builder},-(${ + (await args.fn(args.pt.arguments[1])).builder + }))${args.colAlias}` + ), + }; }, MID: 'SUBSTR', FLOAT: async (args: MapFnArgs) => { - return args.knex - .raw( - `CAST(${(await args.fn(args.pt.arguments[0])).builder} as FLOAT)${ - args.colAlias - }` - ) - .wrap('(', ')'); + return { + builder: args.knex + .raw( + `CAST(${(await args.fn(args.pt.arguments[0])).builder} as FLOAT)${ + args.colAlias + }` + ) + .wrap('(', ')'), + }; }, DATEADD: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { const dateIN = (await fn(pt.arguments[1])).builder; - return knex.raw( - `CASE + return { + builder: knex.raw( + `CASE WHEN ${(await fn(pt.arguments[0])).builder} LIKE '%:%' THEN STRFTIME('%Y-%m-%d %H:%M', DATETIME(DATETIME(${fn( pt.arguments[0] )}, 'localtime'), ${dateIN > 0 ? '+' : ''}${ - (await fn(pt.arguments[1])).builder - } || ' ${String((await fn(pt.arguments[2])).builder).replace( - /["']/g, - '' - )}')) + (await fn(pt.arguments[1])).builder + } || ' ${String((await fn(pt.arguments[2])).builder).replace( + /["']/g, + '' + )}')) ELSE DATE(DATETIME(${(await fn(pt.arguments[0])).builder}, 'localtime'), ${dateIN > 0 ? '+' : ''}${ - (await fn(pt.arguments[1])).builder - } || ' ${String((await fn(pt.arguments[2])).builder).replace( - /["']/g, - '' - )}') + (await fn(pt.arguments[1])).builder + } || ' ${String((await fn(pt.arguments[2])).builder).replace( + /["']/g, + '' + )}') END${colAlias}` - ); + ), + }; }, DATETIME_DIFF: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { let datetime_expr1 = (await fn(pt.arguments[0])).builder; @@ -174,54 +188,60 @@ const sqlite3 = { default: sql = ''; } - return knex.raw(`${sql} ${colAlias}`); + return { builder: knex.raw(`${sql} ${colAlias}`) }; }, WEEKDAY: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { // strftime('%w', date) - day of week 0 - 6 with Sunday == 0 // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday - return knex.raw( - `(strftime('%w', ${ - pt.arguments[0].type === 'Literal' - ? `'${dayjs((await fn(pt.arguments[0])).builder).format( - 'YYYY-MM-DD' - )}'` - : (await fn(pt.arguments[0])).builder - }) - 1 - ${getWeekdayByText( - pt?.arguments[1]?.value - )} % 7 + 7) % 7 ${colAlias}` - ); + return { + builder: knex.raw( + `(strftime('%w', ${ + pt.arguments[0].type === 'Literal' + ? `'${dayjs((await fn(pt.arguments[0])).builder).format( + 'YYYY-MM-DD' + )}'` + : (await fn(pt.arguments[0])).builder + }) - 1 - ${getWeekdayByText( + pt?.arguments[1]?.value + )} % 7 + 7) % 7 ${colAlias}` + ), + }; }, AND: async (args: MapFnArgs) => { - return args.knex.raw( - `CASE WHEN ${args.knex - .raw( - `${( - await Promise.all( - args.pt.arguments.map(async (ar) => - (await args.fn(ar, '', 'AND')).builder.toQuery() + return { + builder: args.knex.raw( + `CASE WHEN ${args.knex + .raw( + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar, '', 'AND')).builder.toQuery() + ) ) - ) - ).join(' AND ')}` - ) - .wrap('(', ')') - .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` - ); + ).join(' AND ')}` + ) + .wrap('(', ')') + .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` + ), + }; }, OR: async (args: MapFnArgs) => { - return args.knex.raw( - `CASE WHEN ${args.knex - .raw( - `${( - await Promise.all( - args.pt.arguments.map(async (ar) => - (await args.fn(ar, '', 'OR')).builder.toQuery() + return { + builder: args.knex.raw( + `CASE WHEN ${args.knex + .raw( + `${( + await Promise.all( + args.pt.arguments.map(async (ar) => + (await args.fn(ar, '', 'OR')).builder.toQuery() + ) ) - ) - ).join(' OR ')}` - ) - .wrap('(', ')') - .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` - ); + ).join(' OR ')}` + ) + .wrap('(', ')') + .toQuery()} THEN 1 ELSE 0 END ${args.colAlias}` + ), + }; }, }; From 639d8b21040a4017c62ad2f543a53e9fa51bd8eb Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Fri, 31 Mar 2023 18:12:56 +0530 Subject: [PATCH 10/27] test: mega table formula fix Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- tests/playwright/tests/megaTable.spec.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/playwright/tests/megaTable.spec.ts b/tests/playwright/tests/megaTable.spec.ts index 456b16c36f..b51af51c97 100644 --- a/tests/playwright/tests/megaTable.spec.ts +++ b/tests/playwright/tests/megaTable.spec.ts @@ -16,7 +16,6 @@ const megaTblColumns = [ { type: 'Checkbox', count: 3 }, { type: 'SingleSelect', count: 3 }, { type: 'MultiSelect', count: 3 }, - { type: 'Formula', count: 0 }, ]; const megaTblRows = 50000; const bulkInsertAfterRows = 1000; @@ -25,7 +24,7 @@ test.describe.serial('Test table', () => { let context: any; test.beforeEach(async ({ page }) => { - context = await setup({ page, isEmptyProject: true }); + context = await setup({ page }); api = new Api({ baseURL: `http://localhost:8080/`, @@ -63,8 +62,6 @@ test.describe.serial('Test table', () => { }; if (megaTblColumns[i].type === 'SingleSelect' || megaTblColumns[i].type === 'MultiSelect') { column['dtxp'] = "'jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec'"; - } else if (megaTblColumns[i].type === 'Formula') { - column['formula_raw'] = '{Id}'; } table_1_columns.push(column); } @@ -78,6 +75,13 @@ test.describe.serial('Test table', () => { columns: table_1_columns, }); + table_1 = await api.dbTableColumn.create(table_1.id, { + column_name: 'Formula', + title: 'Formula', + uidt: UITypes.Formula, + formula_raw: '{SingleLineText}', + }); + const table_1_rows = []; for (let rowCnt = 0; rowCnt < megaTblRows; rowCnt++) { const row = { From 8653ff5ee175124536a21cd2814d7052661bb1f0 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 1 Apr 2023 00:36:26 +0530 Subject: [PATCH 11/27] fix(nocodb): remove extra `.builder` Signed-off-by: Pranav C --- .../lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts index ad38851993..6eef4c5cad 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts @@ -14,7 +14,7 @@ const mssql = { for (const [i, arg] of Object.entries(args.pt.arguments)) { if (+i === args.pt.arguments.length - 1) { query += args.knex - .raw(`\n\tElse ${(await args.fn(arg)).builder.builder.toQuery()}`) + .raw(`\n\tElse ${(await args.fn(arg)).builder.toQuery()}`) .toQuery(); } else { query += args.knex From 1f4465b1de641dff856f4936345c198cefd83992 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 1 Apr 2023 00:54:48 +0530 Subject: [PATCH 12/27] feat(gui): replace ClampText with css to improve performance of grid view Signed-off-by: Pranav C --- .../nc-gui/components/cell/ClampedText.vue | 37 +++++-------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/packages/nc-gui/components/cell/ClampedText.vue b/packages/nc-gui/components/cell/ClampedText.vue index 515a95bed3..4a541268e2 100644 --- a/packages/nc-gui/components/cell/ClampedText.vue +++ b/packages/nc-gui/components/cell/ClampedText.vue @@ -3,35 +3,18 @@ const props = defineProps<{ value?: string | number | null lines?: number }>() - -const wrapper = ref() - -const key = ref(0) - -const debouncedRefresh = useDebounceFn(() => { - key.value++ -}, 500) - -onMounted(() => { - const observer = new ResizeObserver(() => { - debouncedRefresh() - }) - - observer.observe(wrapper.value) -}) From 86b729ad7ecba2003ace6602904b2a3c0ee7b255 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 1 Apr 2023 16:26:24 +0530 Subject: [PATCH 13/27] refactor(gui): render ant select component only when cell is active or in editable state Signed-off-by: Pranav C --- .../nc-gui/components/cell/MultiSelect.vue | 24 ++++++++++++++++++ .../nc-gui/components/cell/SingleSelect.vue | 25 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/packages/nc-gui/components/cell/MultiSelect.vue b/packages/nc-gui/components/cell/MultiSelect.vue index ebef31bd75..3a3d5f56fe 100644 --- a/packages/nc-gui/components/cell/MultiSelect.vue +++ b/packages/nc-gui/components/cell/MultiSelect.vue @@ -313,11 +313,35 @@ const handleClose = (e: MouseEvent) => { } useEventListener(document, 'click', handleClose, true) + +// todo: maintain order +const selectedOpts = computed(() => { + return options.value.filter((o) => vModel.value.includes(o.value!)) +})