Merge pull request #9532 from titouv/feat/json-formula

feat: Implement formulas for interacting with JSON

packages/nocodb-sdk/src/lib/formulaHelpers.ts

@@ -1379,6 +1379,24 @@ export const formulas: Record<string, FormulaMeta> = {
     docsUrl:
       'https://docs.nocodb.com/fields/field-types/formula/numeric-functions#value',
   },
+  JSON_EXTRACT: {
+    docsUrl:
+      'https://docs.nocodb.com/fields/field-types/formula/json-functions#json_extract',
+    validation: {
+      args: {
+        min: 2,
+        max: 2,
+        type: [FormulaDataTypes.STRING, FormulaDataTypes.STRING],
+      },
+    },
+    description: 'Extracts a value from a JSON string using a jq-like syntax',
+    syntax: 'JSON_EXTRACT(json_string, path)',
+    examples: [
+      'JSON_EXTRACT(\'{"a": {"b": "c"}}\', \'.a.b\') => "c"',
+      "JSON_EXTRACT({json_column}, '.key')",
+    ],
+    returnType: FormulaDataTypes.STRING,
+  },
   // Disabling these functions for now; these act as alias for CreatedAt & UpdatedAt fields;
   // Issue: Error noticed if CreatedAt & UpdatedAt fields are removed from the table after creating these formulas
   //

packages/nocodb/src/db/functionMappings/mssql.ts

@@ -127,7 +127,7 @@ const mssql = {
         FORMAT(DATEADD(${String((await fn(pt.arguments[2])).builder).replace(
           /["']/g,
           '',
-        )}, 
+        )},
         ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${
           (await fn(pt.arguments[0])).builder
         }), 'yyyy-MM-dd HH:mm:ss')
@@ -135,7 +135,7 @@ const mssql = {
        FORMAT(DATEADD(${String((await fn(pt.arguments[2])).builder).replace(
          /["']/g,
          '',
-       )}, 
+       )},
        ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${
           (await fn(pt.arguments[0])).builder
         }), 'yyyy-MM-dd')
@@ -209,6 +209,17 @@ const mssql = {
       ),
     };
   },
+  JSON_EXTRACT: async ({ fn, knex, pt, colAlias }: MapFnArgs) => {
+    return {
+      builder: knex.raw(
+        `CASE WHEN ISJSON(${
+          (await fn(pt.arguments[0])).builder
+        }) = 1 THEN JSON_VALUE(${(await fn(pt.arguments[0])).builder}, ${
+          (await fn(pt.arguments[1])).builder
+        }) ELSE NULL END${colAlias}`,
+      ),
+    };
+  },
 };
 
 export default mssql;

packages/nocodb/src/db/functionMappings/mysql.ts

@@ -171,6 +171,19 @@ END) ${colAlias}`,
       ),
     };
   },
+  JSON_EXTRACT: async ({ fn, knex, pt, colAlias }: MapFnArgs) => {
+    return {
+      builder: knex.raw(
+        `CASE WHEN JSON_VALID(${
+          (await fn(pt.arguments[0])).builder
+        }) = 1 THEN JSON_EXTRACT(${
+          (await fn(pt.arguments[0])).builder
+        }, CONCAT('$', ${
+          (await fn(pt.arguments[1])).builder
+        })) ELSE NULL END${colAlias}`,
+      ),
+    };
+  },
 };
 
 export default mysql2;

packages/nocodb/src/db/functionMappings/pg.ts

@@ -60,7 +60,7 @@ const pg = {
       builder: knex.raw(
         `(${(await fn(pt.arguments[0])).builder})${
           pt.arguments[0].dataType !== FormulaDataTypes.DATE ? '::DATE' : ''
-        } + (${(await fn(pt.arguments[1])).builder} || 
+        } + (${(await fn(pt.arguments[1])).builder} ||
       '${String((await fn(pt.arguments[2])).builder).replace(
         /["']/g,
         '',
@@ -94,17 +94,17 @@ const pg = {
         break;
       case 'month':
         sql = `(
-                DATE_PART('year', ${datetime_expr1}::TIMESTAMP) - 
+                DATE_PART('year', ${datetime_expr1}::TIMESTAMP) -
                 DATE_PART('year', ${datetime_expr2}::TIMESTAMP)
                ) * 12 + (
-                DATE_PART('month', ${datetime_expr1}::TIMESTAMP) - 
+                DATE_PART('month', ${datetime_expr1}::TIMESTAMP) -
                 DATE_PART('month', ${datetime_expr2}::TIMESTAMP)
                )`;
         break;
       case 'quarter':
-        sql = `((EXTRACT(QUARTER FROM ${datetime_expr1}::TIMESTAMP) + 
-                    DATE_PART('year', AGE(${datetime_expr1}, '1900/01/01')) * 4) - 1) - 
-                ((EXTRACT(QUARTER FROM ${datetime_expr2}::TIMESTAMP) + 
+        sql = `((EXTRACT(QUARTER FROM ${datetime_expr1}::TIMESTAMP) +
+                    DATE_PART('year', AGE(${datetime_expr1}, '1900/01/01')) * 4) - 1) -
+                ((EXTRACT(QUARTER FROM ${datetime_expr2}::TIMESTAMP) +
                     DATE_PART('year', AGE(${datetime_expr2}, '1900/01/01')) * 4) - 1)`;
         break;
       case 'year':
@@ -305,7 +305,7 @@ const pg = {
 
     return {
       builder: knex.raw(
-        `CASE 
+        `CASE
   WHEN ${value} IS NULL OR REGEXP_REPLACE(${value}::TEXT, '[^\\d.]+', '', 'g') IN ('.', '') OR LENGTH(REGEXP_REPLACE(${value}::TEXT, '[^.]+', '', 'g')) > 1 THEN NULL
   WHEN LENGTH(REGEXP_REPLACE(${value}::TEXT, '[^%]', '','g')) > 0 THEN POW(-1, LENGTH(REGEXP_REPLACE(${value}::TEXT, '[^-]','', 'g'))) * (REGEXP_REPLACE(${value}::TEXT, '[^\\d.]+', '', 'g'))::NUMERIC / 100
   ELSE POW(-1, LENGTH(REGEXP_REPLACE(${value}::TEXT, '[^-]', '', 'g'))) * (REGEXP_REPLACE(${value}::TEXT, '[^\\d.]+', '', 'g'))::NUMERIC
@@ -359,6 +359,19 @@ END ${colAlias}`,
       ),
     };
   },
+  JSON_EXTRACT: async ({ fn, knex, pt, colAlias }: MapFnArgs) => {
+    return {
+      builder: knex.raw(
+        `CASE WHEN (${
+          (await fn(pt.arguments[0])).builder
+        })::jsonb IS NOT NULL THEN jsonb_path_query_first((${
+          (await fn(pt.arguments[0])).builder
+        })::jsonb, CONCAT('$', ${
+          (await fn(pt.arguments[1])).builder
+        })::jsonpath) ELSE NULL END${colAlias}`,
+      ),
+    };
+  },
 };
 
 export default pg;

packages/nocodb/src/db/functionMappings/sqlite.ts

@@ -244,6 +244,19 @@ const sqlite3 = {
       ),
     };
   },
+  async JSON_EXTRACT(args: MapFnArgs) {
+    return {
+      builder: args.knex.raw(
+        `CASE WHEN json_valid(${
+          (await args.fn(args.pt.arguments[0])).builder
+        }) = 1 THEN json_extract(${
+          (await args.fn(args.pt.arguments[0])).builder
+        }, CONCAT('$', ${
+          (await args.fn(args.pt.arguments[1])).builder
+        })) ELSE NULL END${args.colAlias}`,
+      ),
+    };
+  },
 };
 
 export default sqlite3;

packages/nocodb/src/helpers/catchError.ts

@@ -636,7 +636,14 @@ const errorHelpers: {
     code: 400,
   },
   [NcErrorType.FORMULA_ERROR]: {
-    message: (message: string) => `Formula error: ${message}`,
+    message: (message: string) => {
+      // try to extract db error - Experimental
+      if (message.includes(' - ')) {
+        const [_, dbError] = message.split(' - ');
+        return `Formula error: ${dbError}`;
+      }
+      return `Formula error: ${message}`;
+    },
     code: 400,
   },
   [NcErrorType.PERMISSION_DENIED]: {

packages/nocodb/src/services/columns.service.ts

@@ -389,7 +389,7 @@ export class ColumnsService {
             );
           } catch (e) {
             console.error(e);
-            NcError.badRequest('Invalid Formula');
+            throw e;
           }
 
           await Column.update(context, column.id, {
@@ -1843,7 +1843,7 @@ export class ColumnsService {
           );
         } catch (e) {
           console.error(e);
-          NcError.badRequest('Invalid Formula');
+          throw e;
         }
 
         await Column.insert(context, {