From 3ce53b879167e8fe62c50535d5082697a89de04d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:58 +0000 Subject: [PATCH] chore: add jest unit test in nocodb-sdk --- packages/nocodb-sdk/jest.config.js | 5 + .../nocodb-sdk/src/lib/formulaHelpers.spec.ts | 6 + packages/nocodb-sdk/src/lib/formulaHelpers.ts | 528 ++++++++++++++++++ packages/nocodb-sdk/tsconfig.json | 2 +- 4 files changed, 540 insertions(+), 1 deletion(-) create mode 100644 packages/nocodb-sdk/jest.config.js create mode 100644 packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts diff --git a/packages/nocodb-sdk/jest.config.js b/packages/nocodb-sdk/jest.config.js new file mode 100644 index 0000000000..b413e106db --- /dev/null +++ b/packages/nocodb-sdk/jest.config.js @@ -0,0 +1,5 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', +}; \ No newline at end of file diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts new file mode 100644 index 0000000000..6fce60edb8 --- /dev/null +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts @@ -0,0 +1,6 @@ +describe('auth', () => { + it('Formula parsing and type validation', async () => { + const response = {userId: 'fakeUserId'}; + expect(response).toEqual({ userId: 'fakeUserId' }); + }); +}); diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 5a28417d84..8d5cbcbd15 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1,6 +1,7 @@ import jsep from 'jsep'; import { ColumnType } from './Api'; +import {UITypes} from "../../build/main"; export const jsepCurlyHook = { name: 'curly', @@ -189,3 +190,530 @@ function escapeLiteral(v: string) { .replace(/'/g, `\\'`) ); } + + + + +function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = new Set()) { + let returnType: formulaTypes + if (parsedTree.type === JSEPNode.CALL_EXP) { + const calleeName = parsedTree.callee.name.toUpperCase() + // validate function name + if (!availableFunctions.includes(calleeName)) { + errors.add(t('msg.formula.functionNotAvailable', { function: calleeName })) + } + // validate arguments + const validation = formulas[calleeName] && formulas[calleeName].validation + if (validation && validation.args) { + if (validation.args.rqd !== undefined && validation.args.rqd !== parsedTree.arguments.length) { + errors.add(t('msg.formula.requiredArgumentsFormula', { requiredArguments: validation.args.rqd, calleeName })) + } else if (validation.args.min !== undefined && validation.args.min > parsedTree.arguments.length) { + errors.add( + t('msg.formula.minRequiredArgumentsFormula', { + minRequiredArguments: validation.args.min, + calleeName, + }), + ) + } else if (validation.args.max !== undefined && validation.args.max < parsedTree.arguments.length) { + errors.add( + t('msg.formula.maxRequiredArgumentsFormula', { + maxRequiredArguments: validation.args.max, + calleeName, + }), + ) + } + } + + parsedTree.arguments.map((arg: Record) => validateAgainstMeta(arg, errors)) + + // get args type and validate + const validateResult = parsedTree.arguments.map((arg) => { + return validateAgainstMeta(arg, errors, typeErrors) + }) + + const argsTypes = validateResult.map((v: any) => v.returnType); + + if (typeof validateResult[0].returnType === 'function') { + returnType = formulas[calleeName].returnType(argsTypes) + } else if (validateResult[0]) { + returnType = formulas[calleeName].returnType + } + + // validate data type + if (parsedTree.callee.type === JSEPNode.IDENTIFIER) { + const expectedType = formulas[calleeName.toUpperCase()].type + + if (expectedType === formulaTypes.NUMERIC) { + if (calleeName === 'WEEKDAY') { + // parsedTree.arguments[0] = date + validateAgainstType( + parsedTree.arguments[0], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate')) + } + }, + typeErrors, + ) + // parsedTree.arguments[1] = startDayOfWeek (optional) + validateAgainstType( + parsedTree.arguments[1], + formulaTypes.STRING, + (v: any) => { + if ( + typeof v !== 'string' || + !['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'].includes(v.toLowerCase()) + ) { + typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate')) + } + }, + typeErrors, + ) + } else { + parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors, argsTypes)) + } + } else if (expectedType === formulaTypes.DATE) { + if (calleeName === 'DATEADD') { + // parsedTree.arguments[0] = date + validateAgainstType( + parsedTree.arguments[0], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.firstParamDateAddHaveDate')) + } + }, + typeErrors, + ) + // parsedTree.arguments[1] = numeric + validateAgainstType( + parsedTree.arguments[1], + formulaTypes.NUMERIC, + (v: any) => { + if (typeof v !== 'number') { + typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber')) + } + }, + typeErrors, + ) + // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"] + validateAgainstType( + parsedTree.arguments[2], + formulaTypes.STRING, + (v: any) => { + if (!['day', 'week', 'month', 'year'].includes(v)) { + typeErrors.add(typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate'))) + } + }, + typeErrors, + ) + } else if (calleeName === 'DATETIME_DIFF') { + // parsedTree.arguments[0] = date + validateAgainstType( + parsedTree.arguments[0], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate')) + } + }, + typeErrors, + ) + // parsedTree.arguments[1] = date + validateAgainstType( + parsedTree.arguments[1], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate')) + } + }, + typeErrors, + ) + // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"] + validateAgainstType( + parsedTree.arguments[2], + formulaTypes.STRING, + (v: any) => { + if ( + ![ + 'milliseconds', + 'ms', + 'seconds', + 's', + 'minutes', + 'm', + 'hours', + 'h', + 'days', + 'd', + 'weeks', + 'w', + 'months', + 'M', + 'quarters', + 'Q', + 'years', + 'y', + ].includes(v) + ) { + typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate')) + } + }, + typeErrors, + ) + } + } + } + + errors = new Set([...errors, ...typeErrors]) + } else if (parsedTree.type === JSEPNode.IDENTIFIER) { + if (supportedColumns.value.filter((c) => !column || column.value?.id !== c.id).every((c) => c.title !== parsedTree.name)) { + errors.add( + t('msg.formula.columnNotAvailable', { + columnName: parsedTree.name, + }), + ) + } + + // check circular reference + // e.g. formula1 -> formula2 -> formula1 should return circular reference error + + // get all formula columns excluding itself + const formulaPaths = supportedColumns.value + .filter((c) => c.id !== column.value?.id && c.uidt === UITypes.Formula) + .reduce((res: Record[], c: Record) => { + // in `formula`, get all the (unique) target neighbours + // i.e. all column id (e.g. cl_xxxxxxxxxxxxxx) with formula type + const neighbours = [ + ...new Set( + (c.colOptions.formula.match(/cl_\w{14}/g) || []).filter( + (colId: string) => + supportedColumns.value.filter((col: ColumnType) => col.id === colId && col.uidt === UITypes.Formula).length, + ), + ), + ] + if (neighbours.length > 0) { + // e.g. formula column 1 -> [formula column 2, formula column3] + res.push({ [c.id]: neighbours }) + } + return res + }, []) + // include target formula column (i.e. the one to be saved if applicable) + const targetFormulaCol = supportedColumns.value.find( + (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula, + ) + + if (targetFormulaCol && column.value?.id) { + formulaPaths.push({ + [column.value?.id as string]: [targetFormulaCol.id], + }) + } + const vertices = formulaPaths.length + if (vertices > 0) { + // perform kahn's algo for cycle detection + const adj = new Map() + const inDegrees = new Map() + // init adjacency list & indegree + + for (const [_, v] of Object.entries(formulaPaths)) { + const src = Object.keys(v)[0] + const neighbours = v[src] + inDegrees.set(src, inDegrees.get(src) || 0) + for (const neighbour of neighbours) { + adj.set(src, (adj.get(src) || new Set()).add(neighbour)) + inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1) + } + } + const queue: string[] = [] + // put all vertices with in-degree = 0 (i.e. no incoming edges) to queue + inDegrees.forEach((inDegree, col) => { + if (inDegree === 0) { + // in-degree = 0 means we start traversing from this node + queue.push(col) + } + }) + // init count of visited vertices + let visited = 0 + // BFS + while (queue.length !== 0) { + // remove a vertex from the queue + const src = queue.shift() + // if this node has neighbours, increase visited by 1 + const neighbours = adj.get(src) || new Set() + if (neighbours.size > 0) { + visited += 1 + } + // iterate each neighbouring nodes + neighbours.forEach((neighbour: string) => { + // decrease in-degree of its neighbours by 1 + inDegrees.set(neighbour, inDegrees.get(neighbour) - 1) + // if in-degree becomes 0 + if (inDegrees.get(neighbour) === 0) { + // then put the neighboring node to the queue + queue.push(neighbour) + } + }) + } + // vertices not same as visited = cycle found + if (vertices !== visited) { + errors.add(t('msg.formula.cantSaveCircularReference')) + } + } + } else if (parsedTree.type === JSEPNode.BINARY_EXP) { + if (!availableBinOps.includes(parsedTree.operator)) { + errors.add(t('msg.formula.operationNotAvailable', { operation: parsedTree.operator })) + } + validateAgainstMeta(parsedTree.left, errors) + validateAgainstMeta(parsedTree.right, errors) + + // todo: type extraction for binary exps + returnType = formulaTypes.NUMERIC + } else if (parsedTree.type === JSEPNode.LITERAL || parsedTree.type === JSEPNode.UNARY_EXP) { + if (parsedTree.type === JSEPNode.LITERAL) { + if (typeof parsedTree.value === 'number') { + returnType = formulaTypes.NUMERIC + } else if (typeof parsedTree.value === 'string') { + returnType = formulaTypes.STRING + } else if (typeof parsedTree.value === 'boolean') { + returnType = formulaTypes.BOOLEAN + } else { + returnType = formulaTypes.STRING + } + } + // do nothing + } else if (parsedTree.type === JSEPNode.COMPOUND) { + if (parsedTree.body.length) { + errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')) + } + } else { + errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')) + } + return { errors, returnType } +} + +function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set(), argTypes: formulaTypes = []) { + let type + if (parsedTree === false || typeof parsedTree === 'undefined') { + return typeErrors + } + if (parsedTree.type === JSEPNode.LITERAL) { + if (typeof func === 'function') { + func(parsedTree.value) + } else if (expectedType === formulaTypes.NUMERIC) { + if (typeof parsedTree.value !== 'number') { + typeErrors.add(t('msg.formula.numericTypeIsExpected')) + } else { + type = formulaTypes.NUMERIC + } + } else if (expectedType === formulaTypes.STRING) { + if (typeof parsedTree.value !== 'string') { + typeErrors.add(t('msg.formula.stringTypeIsExpected')) + } else { + type = formulaTypes.STRING + } + } + } else if (parsedTree.type === JSEPNode.IDENTIFIER) { + const col = supportedColumns.value.find((c) => c.title === parsedTree.name) + + if (col === undefined) { + return + } + + if (col.uidt === UITypes.Formula) { + const foundType = getRootDataType(jsep(col.colOptions?.formula_raw)) + type = foundType + if (foundType === 'N/A') { + typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: col.title })) + } else if (expectedType !== foundType) { + typeErrors.add( + t('msg.formula.typeIsExpectedButFound', { + type: expectedType, + found: foundType, + }), + ) + } + } else { + switch (col.uidt) { + // string + case UITypes.SingleLineText: + case UITypes.LongText: + case UITypes.MultiSelect: + case UITypes.SingleSelect: + case UITypes.PhoneNumber: + case UITypes.Email: + case UITypes.URL: + if (expectedType !== formulaTypes.STRING) { + typeErrors.add( + t('msg.formula.columnWithTypeFoundButExpected', { + columnName: parsedTree.name, + columnType: formulaTypes.STRING, + expectedType, + }), + ) + } + type = formulaTypes.STRING + break + + // numeric + case UITypes.Year: + case UITypes.Number: + case UITypes.Decimal: + case UITypes.Rating: + case UITypes.Count: + case UITypes.AutoNumber: + case UITypes.Currency: + if (expectedType !== formulaTypes.NUMERIC) { + typeErrors.add( + t('msg.formula.columnWithTypeFoundButExpected', { + columnName: parsedTree.name, + columnType: formulaTypes.NUMERIC, + expectedType, + }), + ) + } + type = formulaTypes.NUMERIC + break + + // date + case UITypes.Date: + case UITypes.DateTime: + case UITypes.CreateTime: + case UITypes.LastModifiedTime: + if (expectedType !== formulaTypes.DATE) { + typeErrors.add( + t('msg.formula.columnWithTypeFoundButExpected', { + columnName: parsedTree.name, + columnType: formulaTypes.DATE, + expectedType, + }), + ) + } + type = formulaTypes.DATE + break + + // not supported + case UITypes.ForeignKey: + case UITypes.Attachment: + case UITypes.ID: + case UITypes.Time: + case UITypes.Percent: + case UITypes.Duration: + case UITypes.Rollup: + case UITypes.Lookup: + case UITypes.Barcode: + case UITypes.Button: + case UITypes.Checkbox: + case UITypes.Collaborator: + case UITypes.QrCode: + default: + typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: parsedTree.name })) + break + } + } + } else if (parsedTree.type === JSEPNode.UNARY_EXP || parsedTree.type === JSEPNode.BINARY_EXP) { + if (expectedType !== formulaTypes.NUMERIC) { + // parsedTree.name won't be available here + typeErrors.add( + t('msg.formula.typeIsExpectedButFound', { + type: formulaTypes.NUMERIC, + found: expectedType, + }), + ) + } + + type = formulaTypes.NUMERIC + } else if (parsedTree.type === JSEPNode.CALL_EXP) { + const calleeName = parsedTree.callee.name.toUpperCase() + if (formulas[calleeName]?.type && expectedType !== formulas[calleeName].type) { + typeErrors.add( + t('msg.formula.typeIsExpectedButFound', { + type: expectedType, + found: formulas[calleeName].type, + }), + ) + } + // todo: derive type from returnType + type = formulas[calleeName]?.type + } + return { type, typeErrors } +} + +function getRootDataType(parsedTree: any): any { + // given a parse tree, return the data type of it + if (parsedTree.type === JSEPNode.CALL_EXP) { + return formulas[parsedTree.callee.name.toUpperCase()].type + } else if (parsedTree.type === JSEPNode.IDENTIFIER) { + const col = supportedColumns.value.find((c) => c.title === parsedTree.name) as Record + if (col?.uidt === UITypes.Formula) { + return getRootDataType(jsep(col?.formula_raw)) + } else { + switch (col?.uidt) { + // string + case UITypes.SingleLineText: + case UITypes.LongText: + case UITypes.MultiSelect: + case UITypes.SingleSelect: + case UITypes.PhoneNumber: + case UITypes.Email: + case UITypes.URL: + return formulaTypes.STRING + + // numeric + case UITypes.Year: + case UITypes.Number: + case UITypes.Decimal: + case UITypes.Rating: + case UITypes.Count: + case UITypes.AutoNumber: + return formulaTypes.NUMERIC + + // date + case UITypes.Date: + case UITypes.DateTime: + case UITypes.CreateTime: + case UITypes.LastModifiedTime: + return formulaTypes.DATE + + // not supported + case UITypes.ForeignKey: + case UITypes.Attachment: + case UITypes.ID: + case UITypes.Time: + case UITypes.Currency: + case UITypes.Percent: + case UITypes.Duration: + case UITypes.Rollup: + case UITypes.Lookup: + case UITypes.Barcode: + case UITypes.Button: + case UITypes.Checkbox: + case UITypes.Collaborator: + case UITypes.QrCode: + default: + return 'N/A' + } + } + } else if (parsedTree.type === JSEPNode.BINARY_EXP || parsedTree.type === JSEPNode.UNARY_EXP) { + return formulaTypes.NUMERIC + } else if (parsedTree.type === JSEPNode.LITERAL) { + return typeof parsedTree.value + } else { + return 'N/A' + } +} + +function isCurlyBracketBalanced() { + // count number of opening curly brackets and closing curly brackets + const cntCurlyBrackets = (formulaRef.value.$el.value.match(/\{|}/g) || []).reduce( + (acc: Record, cur: number) => { + acc[cur] = (acc[cur] || 0) + 1 + return acc + }, + {}, + ) + return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0) +} + + diff --git a/packages/nocodb-sdk/tsconfig.json b/packages/nocodb-sdk/tsconfig.json index 8af501e5e0..0bd395cc39 100644 --- a/packages/nocodb-sdk/tsconfig.json +++ b/packages/nocodb-sdk/tsconfig.json @@ -38,7 +38,7 @@ // "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */, "lib": ["es2017","dom"], - "types": [], + "types": ["jest", "node"], "typeRoots": ["node_modules/@types", "src/types"], "baseUrl": "./src", "paths": {