From b84ce670602057b81756c8f110f279498cd3dbaf Mon Sep 17 00:00:00 2001 From: Verhille Date: Sun, 22 Sep 2024 18:05:55 +0200 Subject: [PATCH 1/5] feat: start implementing JSON_EXTRACT formula --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 109 ++++++++++-------- .../nocodb/src/db/functionMappings/sqlite.ts | 13 +++ 2 files changed, 77 insertions(+), 45 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 3065756c94..abf4137776 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -70,7 +70,7 @@ function validateDateWithUnknownFormat(v: string) { export async function substituteColumnAliasWithIdInFormula( formula, - columns: ColumnType[] + columns: ColumnType[], ) { const substituteId = async (pt: any) => { if (pt.type === 'CallExpression') { @@ -85,7 +85,7 @@ export async function substituteColumnAliasWithIdInFormula( (c) => c.id === colNameOrId || c.column_name === colNameOrId || - c.title === colNameOrId + c.title === colNameOrId, ); pt.name = '{' + column.id + '}'; } else if (pt.type === 'BinaryExpression') { @@ -118,7 +118,7 @@ export enum FormulaErrorType { export function substituteColumnIdWithAliasInFormula( formula, columns: ColumnType[], - rawFormula? + rawFormula?, ) { const substituteId = (pt: any, ptRaw?: any) => { if (pt.type === 'CallExpression') { @@ -134,7 +134,7 @@ export function substituteColumnIdWithAliasInFormula( (c) => c.id === colNameOrId || c.column_name === colNameOrId || - c.title === colNameOrId + c.title === colNameOrId, ); pt.name = column?.title || ptRaw?.name || pt?.name; } else if (pt.type === 'BinaryExpression') { @@ -331,7 +331,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.firstParamDateAddHaveDate' }, - 'First parameter of DATEADD should be a date' + 'First parameter of DATEADD should be a date', ); } } @@ -341,20 +341,20 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.secondParamDateAddHaveNumber' }, - 'Second parameter of DATEADD should be a number' + 'Second parameter of DATEADD should be a number', ); } } if (parsedTree.arguments[2].type === JSEPNode.LITERAL) { if ( !['day', 'week', 'month', 'year'].includes( - parsedTree.arguments[2].value + parsedTree.arguments[2].value, ) ) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.thirdParamDateAddHaveDate' }, - "Third parameter of DATEADD should be one of 'day', 'week', 'month', 'year'" + "Third parameter of DATEADD should be one of 'day', 'week', 'month', 'year'", ); } } @@ -443,7 +443,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.firstParamDateDiffHaveDate' }, - 'First parameter of DATETIME_DIFF should be a date' + 'First parameter of DATETIME_DIFF should be a date', ); } } @@ -453,7 +453,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.secondParamDateDiffHaveDate' }, - 'Second parameter of DATETIME_DIFF should be a date' + 'Second parameter of DATETIME_DIFF should be a date', ); } } @@ -486,7 +486,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.thirdParamDateDiffHaveDate' }, - "Third parameter of DATETIME_DIFF should be one of 'milliseconds', 'ms', 'seconds', 's', 'minutes', 'm', 'hours', 'h', 'days', 'd', 'weeks', 'w', 'months', 'M', 'quarters', 'Q', 'years', 'y'" + "Third parameter of DATETIME_DIFF should be one of 'milliseconds', 'ms', 'seconds', 's', 'minutes', 'm', 'hours', 'h', 'days', 'd', 'weeks', 'w', 'months', 'M', 'quarters', 'Q', 'years', 'y'", ); } } @@ -733,7 +733,7 @@ export const formulas: Record = { calleeName: parsedTree.callee?.name?.toUpperCase(), position: 2, }, - 'The REPEAT function requires a numeric as the parameter at position 2' + 'The REPEAT function requires a numeric as the parameter at position 2', ); } }, @@ -981,7 +981,7 @@ export const formulas: Record = { returnType: (argTypes: FormulaDataTypes[]) => { // extract all return types except NULL, since null can be returned by any type const returnValueTypes = new Set( - argTypes.slice(1).filter((type) => type !== FormulaDataTypes.NULL) + argTypes.slice(1).filter((type) => type !== FormulaDataTypes.NULL), ); // if there are more than one return types or if there is a string return type // return type as string else return the type @@ -1026,7 +1026,7 @@ export const formulas: Record = { returnType: (argTypes: FormulaDataTypes[]) => { // extract all return types except NULL, since null can be returned by any type const returnValueTypes = new Set( - argTypes.slice(2).filter((_, i) => i % 2 === 0) + argTypes.slice(2).filter((_, i) => i % 2 === 0), ); // if there are more than one return types or if there is a string return type @@ -1104,7 +1104,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.firstParamWeekDayHaveDate' }, - 'First parameter of WEEKDAY should be a date' + 'First parameter of WEEKDAY should be a date', ); } } @@ -1130,7 +1130,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.secondParamWeekDayHaveDate' }, - 'Second parameter of WEEKDAY should be day of week string' + 'Second parameter of WEEKDAY should be day of week string', ); } } @@ -1379,6 +1379,24 @@ export const formulas: Record = { docsUrl: 'https://docs.nocodb.com/fields/field-types/formula/numeric-functions#value', }, + JSON_EXTRACT: { + docsUrl: + 'https://docs.nocodb.com/fields/field-types/formula/json-functions#json_extract', + validation: { + args: { + min: 2, + max: 2, + type: [FormulaDataTypes.STRING, FormulaDataTypes.STRING], + }, + }, + description: 'Extracts a value from a JSON string using a jq-like syntax', + syntax: 'JSON_EXTRACT(json_string, path)', + examples: [ + 'JSON_EXTRACT(\'{"a": {"b": "c"}}\', \'.a.b\') => "c"', + "JSON_EXTRACT({json_column}, '.key')", + ], + returnType: FormulaDataTypes.STRING, + }, // Disabling these functions for now; these act as alias for CreatedAt & UpdatedAt fields; // Issue: Error noticed if CreatedAt & UpdatedAt fields are removed from the table after creating these formulas // @@ -1413,7 +1431,7 @@ export class FormulaError extends Error { extra: { [key: string]: any; }, - message: string = 'Formula Error' + message: string = 'Formula Error', ) { super(message); this.type = type; @@ -1511,19 +1529,19 @@ async function extractColumnIdentifierType({ } else { const relationColumnOpt = columns.find( (column) => - column.id === (col.colOptions).fk_relation_column_id + column.id === (col.colOptions).fk_relation_column_id, ); // the value is based on the foreign rollup column type const refTableMeta = await getMeta( (relationColumnOpt.colOptions) - .fk_related_model_id + .fk_related_model_id, ); const refTableColumns = refTableMeta.columns; const childFieldColumn = refTableColumns.find( (column: ColumnType) => - column.id === col.colOptions.fk_rollup_column_id + column.id === col.colOptions.fk_rollup_column_id, ); // extract type and add to res @@ -1534,7 +1552,7 @@ async function extractColumnIdentifierType({ columns: refTableColumns, getMeta, clientOrSqlUi, - }) + }), ); } } @@ -1641,13 +1659,13 @@ export async function validateFormulaAndExtractTreeWithType({ throw new FormulaError( FormulaErrorType.INVALID_FUNCTION_NAME, {}, - `Function ${calleeName} is not available` + `Function ${calleeName} is not available`, ); } else if (sqlUI?.getUnsupportedFnList().includes(calleeName)) { throw new FormulaError( FormulaErrorType.INVALID_FUNCTION_NAME, {}, - `Function ${calleeName} is unavailable for your database` + `Function ${calleeName} is unavailable for your database`, ); } @@ -1666,7 +1684,7 @@ export async function validateFormulaAndExtractTreeWithType({ requiredArguments: validation.args.rqd, calleeName, }, - 'Required arguments missing' + 'Required arguments missing', ); } else if ( validation.args.min !== undefined && @@ -1679,7 +1697,7 @@ export async function validateFormulaAndExtractTreeWithType({ minRequiredArguments: validation.args.min, calleeName, }, - 'Minimum arguments required' + 'Minimum arguments required', ); } else if ( validation.args.max !== undefined && @@ -1692,7 +1710,7 @@ export async function validateFormulaAndExtractTreeWithType({ maxRequiredArguments: validation.args.max, calleeName, }, - 'Maximum arguments missing' + 'Maximum arguments missing', ); } } @@ -1700,7 +1718,7 @@ export async function validateFormulaAndExtractTreeWithType({ const validateResult = (res.arguments = await Promise.all( parsedTree.arguments.map((arg) => { return validateAndExtract(arg); - }) + }), )); const argTypes = validateResult.map((v: any) => v.dataType); @@ -1716,7 +1734,7 @@ export async function validateFormulaAndExtractTreeWithType({ // if type const expectedArgType = Array.isArray( - formulas[calleeName].validation.args.type + formulas[calleeName].validation.args.type, ) ? formulas[calleeName].validation.args.type[i] : formulas[calleeName].validation.args.type; @@ -1730,7 +1748,7 @@ export async function validateFormulaAndExtractTreeWithType({ if (argPt.type === JSEPNode.IDENTIFIER) { const name = columns?.find( - (c) => c.id === argPt.name || c.title === argPt.name + (c) => c.id === argPt.name || c.title === argPt.name, )?.title || argPt.name; throw new FormulaError( @@ -1741,7 +1759,7 @@ export async function validateFormulaAndExtractTreeWithType({ columnType: argPt.dataType, expectedType: expectedArgType, }, - `Field ${name} with ${argPt.dataType} type is found but ${expectedArgType} type is expected` + `Field ${name} with ${argPt.dataType} type is found but ${expectedArgType} type is expected`, ); } else { let key = ''; @@ -1769,7 +1787,7 @@ export async function validateFormulaAndExtractTreeWithType({ }, `${calleeName?.toUpperCase()} requires a ${ type || expectedArgType - } at position ${position}` + } at position ${position}`, ); } } @@ -1786,7 +1804,7 @@ export async function validateFormulaAndExtractTreeWithType({ if (typeof formulas[calleeName].returnType === 'function') { res.dataType = (formulas[calleeName].returnType as any)?.( - argTypes + argTypes, ) as FormulaDataTypes; } else if (formulas[calleeName].returnType) { res.dataType = formulas[calleeName].returnType as FormulaDataTypes; @@ -1802,7 +1820,7 @@ export async function validateFormulaAndExtractTreeWithType({ key: 'msg.formula.columnNotAvailable', columnName: parsedTree.name, }, - `Invalid column name/id ${JSON.stringify(parsedTree.name)} in formula` + `Invalid column name/id ${JSON.stringify(parsedTree.name)} in formula`, ); } @@ -1825,7 +1843,7 @@ export async function validateFormulaAndExtractTreeWithType({ columns, clientOrSqlUi, getMeta, - } + }, )); res.dataType = (formulaRes as any)?.dataType; @@ -1838,7 +1856,7 @@ export async function validateFormulaAndExtractTreeWithType({ columns, getMeta, clientOrSqlUi, - }) + }), ); } } else if (parsedTree.type === JSEPNode.LITERAL) { @@ -1863,7 +1881,7 @@ export async function validateFormulaAndExtractTreeWithType({ throw new FormulaError( FormulaErrorType.NOT_SUPPORTED, {}, - `Unary expression '${parsedTree.operator}' is not supported` + `Unary expression '${parsedTree.operator}' is not supported`, ); } } else if (parsedTree.type === JSEPNode.BINARY_EXP) { @@ -1884,7 +1902,7 @@ export async function validateFormulaAndExtractTreeWithType({ FormulaDataTypes.BOOLEAN, FormulaDataTypes.NULL, FormulaDataTypes.UNKNOWN, - ].includes(r.dataType) + ].includes(r.dataType), ) ) { res.dataType = FormulaDataTypes.STRING; @@ -1896,19 +1914,19 @@ export async function validateFormulaAndExtractTreeWithType({ throw new FormulaError( FormulaErrorType.NOT_SUPPORTED, {}, - 'Bracket notation is not supported' + 'Bracket notation is not supported', ); } else if (parsedTree.type === JSEPNode.ARRAY_EXP) { throw new FormulaError( FormulaErrorType.NOT_SUPPORTED, {}, - 'Array is not supported' + 'Array is not supported', ); } else if (parsedTree.type === JSEPNode.COMPOUND) { throw new FormulaError( FormulaErrorType.NOT_SUPPORTED, {}, - 'Compound statement is not supported' + 'Compound statement is not supported', ); } @@ -1938,9 +1956,9 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { (colId: string) => columns.filter( (col: ColumnType) => - col.id === colId && col.uidt === UITypes.Formula - ).length - ) + col.id === colId && col.uidt === UITypes.Formula, + ).length, + ), ), ]; if (neighbours.length > 0) { @@ -1952,7 +1970,8 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { // include target formula column (i.e. the one to be saved if applicable) const targetFormulaCol = columns.find( - (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula + (c: ColumnType) => + c.title === parsedTree.name && c.uidt === UITypes.Formula, ); if (targetFormulaCol && formulaCol?.id) { @@ -2013,7 +2032,7 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { { key: 'msg.formula.cantSaveCircularReference', }, - 'Circular reference detected' + 'Circular reference detected', ); } } diff --git a/packages/nocodb/src/db/functionMappings/sqlite.ts b/packages/nocodb/src/db/functionMappings/sqlite.ts index f271354393..d57e881c50 100644 --- a/packages/nocodb/src/db/functionMappings/sqlite.ts +++ b/packages/nocodb/src/db/functionMappings/sqlite.ts @@ -244,6 +244,19 @@ const sqlite3 = { ), }; }, + async JSON_EXTRACT(args: MapFnArgs) { + return { + builder: args.knex.raw( + `CASE WHEN json_valid(${ + (await args.fn(args.pt.arguments[0])).builder + }) = 1 THEN json_extract(${ + (await args.fn(args.pt.arguments[0])).builder + }, ${(await args.fn(args.pt.arguments[1])).builder}) ELSE NULL END${ + args.colAlias + }`, + ), + }; + }, }; export default sqlite3; From cb77f071a07e76f2eade9f89a2e7a5c4cdca3b8f Mon Sep 17 00:00:00 2001 From: Verhille Date: Sun, 22 Sep 2024 18:26:06 +0200 Subject: [PATCH 2/5] feat: implement JSON_EXTRACT for mssql, mysql, pg --- .../nocodb/src/db/functionMappings/mssql.ts | 15 +++++++++-- .../nocodb/src/db/functionMappings/mysql.ts | 11 ++++++++ packages/nocodb/src/db/functionMappings/pg.ts | 27 ++++++++++++++----- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/packages/nocodb/src/db/functionMappings/mssql.ts b/packages/nocodb/src/db/functionMappings/mssql.ts index 2b17bfde61..57fd303902 100644 --- a/packages/nocodb/src/db/functionMappings/mssql.ts +++ b/packages/nocodb/src/db/functionMappings/mssql.ts @@ -127,7 +127,7 @@ const mssql = { FORMAT(DATEADD(${String((await fn(pt.arguments[2])).builder).replace( /["']/g, '', - )}, + )}, ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${ (await fn(pt.arguments[0])).builder }), 'yyyy-MM-dd HH:mm:ss') @@ -135,7 +135,7 @@ const mssql = { FORMAT(DATEADD(${String((await fn(pt.arguments[2])).builder).replace( /["']/g, '', - )}, + )}, ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${ (await fn(pt.arguments[0])).builder }), 'yyyy-MM-dd') @@ -209,6 +209,17 @@ const mssql = { ), }; }, + JSON_EXTRACT: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + return { + builder: knex.raw( + `CASE WHEN ISJSON(${ + (await fn(pt.arguments[0])).builder + }) = 1 THEN JSON_VALUE(${(await fn(pt.arguments[0])).builder}, ${ + (await fn(pt.arguments[1])).builder + }) ELSE NULL END${colAlias}`, + ), + }; + }, }; export default mssql; diff --git a/packages/nocodb/src/db/functionMappings/mysql.ts b/packages/nocodb/src/db/functionMappings/mysql.ts index 93d798bde3..6656d41e6b 100644 --- a/packages/nocodb/src/db/functionMappings/mysql.ts +++ b/packages/nocodb/src/db/functionMappings/mysql.ts @@ -171,6 +171,17 @@ END) ${colAlias}`, ), }; }, + JSON_EXTRACT: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + return { + builder: knex.raw( + `CASE WHEN JSON_VALID(${ + (await fn(pt.arguments[0])).builder + }) = 1 THEN JSON_EXTRACT(${(await fn(pt.arguments[0])).builder}, ${ + (await fn(pt.arguments[1])).builder + }) ELSE NULL END${colAlias}`, + ), + }; + }, }; export default mysql2; diff --git a/packages/nocodb/src/db/functionMappings/pg.ts b/packages/nocodb/src/db/functionMappings/pg.ts index 3d877855c3..c6b6ccd78b 100644 --- a/packages/nocodb/src/db/functionMappings/pg.ts +++ b/packages/nocodb/src/db/functionMappings/pg.ts @@ -60,7 +60,7 @@ const pg = { builder: knex.raw( `(${(await fn(pt.arguments[0])).builder})${ pt.arguments[0].dataType !== FormulaDataTypes.DATE ? '::DATE' : '' - } + (${(await fn(pt.arguments[1])).builder} || + } + (${(await fn(pt.arguments[1])).builder} || '${String((await fn(pt.arguments[2])).builder).replace( /["']/g, '', @@ -94,17 +94,17 @@ const pg = { break; case 'month': sql = `( - DATE_PART('year', ${datetime_expr1}::TIMESTAMP) - + DATE_PART('year', ${datetime_expr1}::TIMESTAMP) - DATE_PART('year', ${datetime_expr2}::TIMESTAMP) ) * 12 + ( - DATE_PART('month', ${datetime_expr1}::TIMESTAMP) - + DATE_PART('month', ${datetime_expr1}::TIMESTAMP) - DATE_PART('month', ${datetime_expr2}::TIMESTAMP) )`; break; case 'quarter': - sql = `((EXTRACT(QUARTER FROM ${datetime_expr1}::TIMESTAMP) + - DATE_PART('year', AGE(${datetime_expr1}, '1900/01/01')) * 4) - 1) - - ((EXTRACT(QUARTER FROM ${datetime_expr2}::TIMESTAMP) + + sql = `((EXTRACT(QUARTER FROM ${datetime_expr1}::TIMESTAMP) + + DATE_PART('year', AGE(${datetime_expr1}, '1900/01/01')) * 4) - 1) - + ((EXTRACT(QUARTER FROM ${datetime_expr2}::TIMESTAMP) + DATE_PART('year', AGE(${datetime_expr2}, '1900/01/01')) * 4) - 1)`; break; case 'year': @@ -305,7 +305,7 @@ const pg = { return { builder: knex.raw( - `CASE + `CASE WHEN ${value} IS NULL OR REGEXP_REPLACE(${value}::TEXT, '[^\\d.]+', '', 'g') IN ('.', '') OR LENGTH(REGEXP_REPLACE(${value}::TEXT, '[^.]+', '', 'g')) > 1 THEN NULL WHEN LENGTH(REGEXP_REPLACE(${value}::TEXT, '[^%]', '','g')) > 0 THEN POW(-1, LENGTH(REGEXP_REPLACE(${value}::TEXT, '[^-]','', 'g'))) * (REGEXP_REPLACE(${value}::TEXT, '[^\\d.]+', '', 'g'))::NUMERIC / 100 ELSE POW(-1, LENGTH(REGEXP_REPLACE(${value}::TEXT, '[^-]', '', 'g'))) * (REGEXP_REPLACE(${value}::TEXT, '[^\\d.]+', '', 'g'))::NUMERIC @@ -359,6 +359,19 @@ END ${colAlias}`, ), }; }, + JSON_EXTRACT: async ({ fn, knex, pt, colAlias }: MapFnArgs) => { + return { + builder: knex.raw( + `CASE WHEN (${ + (await fn(pt.arguments[0])).builder + })::jsonb IS NOT NULL THEN (${ + (await fn(pt.arguments[0])).builder + })::jsonb #> ${ + (await fn(pt.arguments[1])).builder + } ELSE NULL END${colAlias}`, + ), + }; + }, }; export default pg; From 2642ddd263eb324278756aacf752c23051fb353c Mon Sep 17 00:00:00 2001 From: Verhille Date: Sun, 22 Sep 2024 18:26:22 +0200 Subject: [PATCH 3/5] fix: prettier --- packages/nocodb-sdk/src/lib/Api.ts | 16 +++- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 91 +++++++++---------- 2 files changed, 60 insertions(+), 47 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/Api.ts b/packages/nocodb-sdk/src/lib/Api.ts index 87c3560f1a..c12e542bcc 100644 --- a/packages/nocodb-sdk/src/lib/Api.ts +++ b/packages/nocodb-sdk/src/lib/Api.ts @@ -765,6 +765,8 @@ export interface FilterType { | 'tomorrow' | ('yesterday' & null) ); + /** Foreign Key to parent column */ + fk_parent_column_id?: StringOrNullType; /** Foreign Key to Column */ fk_column_id?: StringOrNullType; /** Foreign Key to Hook */ @@ -789,6 +791,11 @@ export interface FilterType { base_id?: string; /** The filter value. Can be NULL for some operators. */ value?: any; + /** + * The order of the filter + * @example 1 + */ + order?: number; } /** @@ -7757,7 +7764,13 @@ export class Api< }` */ - read: (viewId: string, params: RequestParams = {}) => + read: ( + viewId: string, + query?: { + includeAllFilters?: boolean; + }, + params: RequestParams = {} + ) => this.request< FilterListType, { @@ -7767,6 +7780,7 @@ export class Api< >({ path: `/api/v1/db/meta/views/${viewId}/filters`, method: 'GET', + query: query, format: 'json', ...params, }), diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index abf4137776..e5c2b4ff1a 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -70,7 +70,7 @@ function validateDateWithUnknownFormat(v: string) { export async function substituteColumnAliasWithIdInFormula( formula, - columns: ColumnType[], + columns: ColumnType[] ) { const substituteId = async (pt: any) => { if (pt.type === 'CallExpression') { @@ -85,7 +85,7 @@ export async function substituteColumnAliasWithIdInFormula( (c) => c.id === colNameOrId || c.column_name === colNameOrId || - c.title === colNameOrId, + c.title === colNameOrId ); pt.name = '{' + column.id + '}'; } else if (pt.type === 'BinaryExpression') { @@ -118,7 +118,7 @@ export enum FormulaErrorType { export function substituteColumnIdWithAliasInFormula( formula, columns: ColumnType[], - rawFormula?, + rawFormula? ) { const substituteId = (pt: any, ptRaw?: any) => { if (pt.type === 'CallExpression') { @@ -134,7 +134,7 @@ export function substituteColumnIdWithAliasInFormula( (c) => c.id === colNameOrId || c.column_name === colNameOrId || - c.title === colNameOrId, + c.title === colNameOrId ); pt.name = column?.title || ptRaw?.name || pt?.name; } else if (pt.type === 'BinaryExpression') { @@ -331,7 +331,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.firstParamDateAddHaveDate' }, - 'First parameter of DATEADD should be a date', + 'First parameter of DATEADD should be a date' ); } } @@ -341,20 +341,20 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.secondParamDateAddHaveNumber' }, - 'Second parameter of DATEADD should be a number', + 'Second parameter of DATEADD should be a number' ); } } if (parsedTree.arguments[2].type === JSEPNode.LITERAL) { if ( !['day', 'week', 'month', 'year'].includes( - parsedTree.arguments[2].value, + parsedTree.arguments[2].value ) ) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.thirdParamDateAddHaveDate' }, - "Third parameter of DATEADD should be one of 'day', 'week', 'month', 'year'", + "Third parameter of DATEADD should be one of 'day', 'week', 'month', 'year'" ); } } @@ -443,7 +443,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.firstParamDateDiffHaveDate' }, - 'First parameter of DATETIME_DIFF should be a date', + 'First parameter of DATETIME_DIFF should be a date' ); } } @@ -453,7 +453,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.secondParamDateDiffHaveDate' }, - 'Second parameter of DATETIME_DIFF should be a date', + 'Second parameter of DATETIME_DIFF should be a date' ); } } @@ -486,7 +486,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.thirdParamDateDiffHaveDate' }, - "Third parameter of DATETIME_DIFF should be one of 'milliseconds', 'ms', 'seconds', 's', 'minutes', 'm', 'hours', 'h', 'days', 'd', 'weeks', 'w', 'months', 'M', 'quarters', 'Q', 'years', 'y'", + "Third parameter of DATETIME_DIFF should be one of 'milliseconds', 'ms', 'seconds', 's', 'minutes', 'm', 'hours', 'h', 'days', 'd', 'weeks', 'w', 'months', 'M', 'quarters', 'Q', 'years', 'y'" ); } } @@ -733,7 +733,7 @@ export const formulas: Record = { calleeName: parsedTree.callee?.name?.toUpperCase(), position: 2, }, - 'The REPEAT function requires a numeric as the parameter at position 2', + 'The REPEAT function requires a numeric as the parameter at position 2' ); } }, @@ -981,7 +981,7 @@ export const formulas: Record = { returnType: (argTypes: FormulaDataTypes[]) => { // extract all return types except NULL, since null can be returned by any type const returnValueTypes = new Set( - argTypes.slice(1).filter((type) => type !== FormulaDataTypes.NULL), + argTypes.slice(1).filter((type) => type !== FormulaDataTypes.NULL) ); // if there are more than one return types or if there is a string return type // return type as string else return the type @@ -1026,7 +1026,7 @@ export const formulas: Record = { returnType: (argTypes: FormulaDataTypes[]) => { // extract all return types except NULL, since null can be returned by any type const returnValueTypes = new Set( - argTypes.slice(2).filter((_, i) => i % 2 === 0), + argTypes.slice(2).filter((_, i) => i % 2 === 0) ); // if there are more than one return types or if there is a string return type @@ -1104,7 +1104,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.firstParamWeekDayHaveDate' }, - 'First parameter of WEEKDAY should be a date', + 'First parameter of WEEKDAY should be a date' ); } } @@ -1130,7 +1130,7 @@ export const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.secondParamWeekDayHaveDate' }, - 'Second parameter of WEEKDAY should be day of week string', + 'Second parameter of WEEKDAY should be day of week string' ); } } @@ -1431,7 +1431,7 @@ export class FormulaError extends Error { extra: { [key: string]: any; }, - message: string = 'Formula Error', + message: string = 'Formula Error' ) { super(message); this.type = type; @@ -1529,19 +1529,19 @@ async function extractColumnIdentifierType({ } else { const relationColumnOpt = columns.find( (column) => - column.id === (col.colOptions).fk_relation_column_id, + column.id === (col.colOptions).fk_relation_column_id ); // the value is based on the foreign rollup column type const refTableMeta = await getMeta( (relationColumnOpt.colOptions) - .fk_related_model_id, + .fk_related_model_id ); const refTableColumns = refTableMeta.columns; const childFieldColumn = refTableColumns.find( (column: ColumnType) => - column.id === col.colOptions.fk_rollup_column_id, + column.id === col.colOptions.fk_rollup_column_id ); // extract type and add to res @@ -1552,7 +1552,7 @@ async function extractColumnIdentifierType({ columns: refTableColumns, getMeta, clientOrSqlUi, - }), + }) ); } } @@ -1659,13 +1659,13 @@ export async function validateFormulaAndExtractTreeWithType({ throw new FormulaError( FormulaErrorType.INVALID_FUNCTION_NAME, {}, - `Function ${calleeName} is not available`, + `Function ${calleeName} is not available` ); } else if (sqlUI?.getUnsupportedFnList().includes(calleeName)) { throw new FormulaError( FormulaErrorType.INVALID_FUNCTION_NAME, {}, - `Function ${calleeName} is unavailable for your database`, + `Function ${calleeName} is unavailable for your database` ); } @@ -1684,7 +1684,7 @@ export async function validateFormulaAndExtractTreeWithType({ requiredArguments: validation.args.rqd, calleeName, }, - 'Required arguments missing', + 'Required arguments missing' ); } else if ( validation.args.min !== undefined && @@ -1697,7 +1697,7 @@ export async function validateFormulaAndExtractTreeWithType({ minRequiredArguments: validation.args.min, calleeName, }, - 'Minimum arguments required', + 'Minimum arguments required' ); } else if ( validation.args.max !== undefined && @@ -1710,7 +1710,7 @@ export async function validateFormulaAndExtractTreeWithType({ maxRequiredArguments: validation.args.max, calleeName, }, - 'Maximum arguments missing', + 'Maximum arguments missing' ); } } @@ -1718,7 +1718,7 @@ export async function validateFormulaAndExtractTreeWithType({ const validateResult = (res.arguments = await Promise.all( parsedTree.arguments.map((arg) => { return validateAndExtract(arg); - }), + }) )); const argTypes = validateResult.map((v: any) => v.dataType); @@ -1734,7 +1734,7 @@ export async function validateFormulaAndExtractTreeWithType({ // if type const expectedArgType = Array.isArray( - formulas[calleeName].validation.args.type, + formulas[calleeName].validation.args.type ) ? formulas[calleeName].validation.args.type[i] : formulas[calleeName].validation.args.type; @@ -1748,7 +1748,7 @@ export async function validateFormulaAndExtractTreeWithType({ if (argPt.type === JSEPNode.IDENTIFIER) { const name = columns?.find( - (c) => c.id === argPt.name || c.title === argPt.name, + (c) => c.id === argPt.name || c.title === argPt.name )?.title || argPt.name; throw new FormulaError( @@ -1759,7 +1759,7 @@ export async function validateFormulaAndExtractTreeWithType({ columnType: argPt.dataType, expectedType: expectedArgType, }, - `Field ${name} with ${argPt.dataType} type is found but ${expectedArgType} type is expected`, + `Field ${name} with ${argPt.dataType} type is found but ${expectedArgType} type is expected` ); } else { let key = ''; @@ -1787,7 +1787,7 @@ export async function validateFormulaAndExtractTreeWithType({ }, `${calleeName?.toUpperCase()} requires a ${ type || expectedArgType - } at position ${position}`, + } at position ${position}` ); } } @@ -1804,7 +1804,7 @@ export async function validateFormulaAndExtractTreeWithType({ if (typeof formulas[calleeName].returnType === 'function') { res.dataType = (formulas[calleeName].returnType as any)?.( - argTypes, + argTypes ) as FormulaDataTypes; } else if (formulas[calleeName].returnType) { res.dataType = formulas[calleeName].returnType as FormulaDataTypes; @@ -1820,7 +1820,7 @@ export async function validateFormulaAndExtractTreeWithType({ key: 'msg.formula.columnNotAvailable', columnName: parsedTree.name, }, - `Invalid column name/id ${JSON.stringify(parsedTree.name)} in formula`, + `Invalid column name/id ${JSON.stringify(parsedTree.name)} in formula` ); } @@ -1843,7 +1843,7 @@ export async function validateFormulaAndExtractTreeWithType({ columns, clientOrSqlUi, getMeta, - }, + } )); res.dataType = (formulaRes as any)?.dataType; @@ -1856,7 +1856,7 @@ export async function validateFormulaAndExtractTreeWithType({ columns, getMeta, clientOrSqlUi, - }), + }) ); } } else if (parsedTree.type === JSEPNode.LITERAL) { @@ -1881,7 +1881,7 @@ export async function validateFormulaAndExtractTreeWithType({ throw new FormulaError( FormulaErrorType.NOT_SUPPORTED, {}, - `Unary expression '${parsedTree.operator}' is not supported`, + `Unary expression '${parsedTree.operator}' is not supported` ); } } else if (parsedTree.type === JSEPNode.BINARY_EXP) { @@ -1902,7 +1902,7 @@ export async function validateFormulaAndExtractTreeWithType({ FormulaDataTypes.BOOLEAN, FormulaDataTypes.NULL, FormulaDataTypes.UNKNOWN, - ].includes(r.dataType), + ].includes(r.dataType) ) ) { res.dataType = FormulaDataTypes.STRING; @@ -1914,19 +1914,19 @@ export async function validateFormulaAndExtractTreeWithType({ throw new FormulaError( FormulaErrorType.NOT_SUPPORTED, {}, - 'Bracket notation is not supported', + 'Bracket notation is not supported' ); } else if (parsedTree.type === JSEPNode.ARRAY_EXP) { throw new FormulaError( FormulaErrorType.NOT_SUPPORTED, {}, - 'Array is not supported', + 'Array is not supported' ); } else if (parsedTree.type === JSEPNode.COMPOUND) { throw new FormulaError( FormulaErrorType.NOT_SUPPORTED, {}, - 'Compound statement is not supported', + 'Compound statement is not supported' ); } @@ -1956,9 +1956,9 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { (colId: string) => columns.filter( (col: ColumnType) => - col.id === colId && col.uidt === UITypes.Formula, - ).length, - ), + col.id === colId && col.uidt === UITypes.Formula + ).length + ) ), ]; if (neighbours.length > 0) { @@ -1970,8 +1970,7 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { // include target formula column (i.e. the one to be saved if applicable) const targetFormulaCol = columns.find( - (c: ColumnType) => - c.title === parsedTree.name && c.uidt === UITypes.Formula, + (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula ); if (targetFormulaCol && formulaCol?.id) { @@ -2032,7 +2031,7 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { { key: 'msg.formula.cantSaveCircularReference', }, - 'Circular reference detected', + 'Circular reference detected' ); } } From a2f98af534be52f12b7b0e9007c239976febb5b0 Mon Sep 17 00:00:00 2001 From: mertmit Date: Thu, 17 Oct 2024 23:03:55 +0300 Subject: [PATCH 4/5] fix: properly throw formula errors Signed-off-by: mertmit --- packages/nocodb/src/helpers/catchError.ts | 9 ++++++++- packages/nocodb/src/services/columns.service.ts | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/helpers/catchError.ts b/packages/nocodb/src/helpers/catchError.ts index 7fc305ec87..1cf89641b6 100644 --- a/packages/nocodb/src/helpers/catchError.ts +++ b/packages/nocodb/src/helpers/catchError.ts @@ -636,7 +636,14 @@ const errorHelpers: { code: 400, }, [NcErrorType.FORMULA_ERROR]: { - message: (message: string) => `Formula error: ${message}`, + message: (message: string) => { + // try to extract db error - Experimental + if (message.includes(' - ')) { + const [_, dbError] = message.split(' - '); + return `Formula error: ${dbError}`; + } + return `Formula error: ${message}`; + }, code: 400, }, [NcErrorType.PERMISSION_DENIED]: { diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index 1668774fad..472b44d4fc 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -388,7 +388,7 @@ export class ColumnsService { ); } catch (e) { console.error(e); - NcError.badRequest('Invalid Formula'); + throw e; } await Column.update(context, column.id, { @@ -1802,7 +1802,7 @@ export class ColumnsService { ); } catch (e) { console.error(e); - NcError.badRequest('Invalid Formula'); + throw e; } await Column.insert(context, { From 5f490bd3eb0e23095632ad8252d9c97466a7146e Mon Sep 17 00:00:00 2001 From: mertmit Date: Thu, 17 Oct 2024 23:05:00 +0300 Subject: [PATCH 5/5] fix: unify formula syntax as documented Signed-off-by: mertmit --- packages/nocodb/src/db/functionMappings/mysql.ts | 6 ++++-- packages/nocodb/src/db/functionMappings/pg.ts | 6 +++--- packages/nocodb/src/db/functionMappings/sqlite.ts | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/nocodb/src/db/functionMappings/mysql.ts b/packages/nocodb/src/db/functionMappings/mysql.ts index 6656d41e6b..9baf831478 100644 --- a/packages/nocodb/src/db/functionMappings/mysql.ts +++ b/packages/nocodb/src/db/functionMappings/mysql.ts @@ -176,9 +176,11 @@ END) ${colAlias}`, builder: knex.raw( `CASE WHEN JSON_VALID(${ (await fn(pt.arguments[0])).builder - }) = 1 THEN JSON_EXTRACT(${(await fn(pt.arguments[0])).builder}, ${ + }) = 1 THEN JSON_EXTRACT(${ + (await fn(pt.arguments[0])).builder + }, CONCAT('$', ${ (await fn(pt.arguments[1])).builder - }) ELSE NULL END${colAlias}`, + })) ELSE NULL END${colAlias}`, ), }; }, diff --git a/packages/nocodb/src/db/functionMappings/pg.ts b/packages/nocodb/src/db/functionMappings/pg.ts index c6b6ccd78b..4385e1875a 100644 --- a/packages/nocodb/src/db/functionMappings/pg.ts +++ b/packages/nocodb/src/db/functionMappings/pg.ts @@ -364,11 +364,11 @@ END ${colAlias}`, builder: knex.raw( `CASE WHEN (${ (await fn(pt.arguments[0])).builder - })::jsonb IS NOT NULL THEN (${ + })::jsonb IS NOT NULL THEN jsonb_path_query_first((${ (await fn(pt.arguments[0])).builder - })::jsonb #> ${ + })::jsonb, CONCAT('$', ${ (await fn(pt.arguments[1])).builder - } ELSE NULL END${colAlias}`, + })::jsonpath) ELSE NULL END${colAlias}`, ), }; }, diff --git a/packages/nocodb/src/db/functionMappings/sqlite.ts b/packages/nocodb/src/db/functionMappings/sqlite.ts index d57e881c50..b90c796afe 100644 --- a/packages/nocodb/src/db/functionMappings/sqlite.ts +++ b/packages/nocodb/src/db/functionMappings/sqlite.ts @@ -251,9 +251,9 @@ const sqlite3 = { (await args.fn(args.pt.arguments[0])).builder }) = 1 THEN json_extract(${ (await args.fn(args.pt.arguments[0])).builder - }, ${(await args.fn(args.pt.arguments[1])).builder}) ELSE NULL END${ - args.colAlias - }`, + }, CONCAT('$', ${ + (await args.fn(args.pt.arguments[1])).builder + })) ELSE NULL END${args.colAlias}`, ), }; },