Merge pull request #4629 from nocodb/feat/formula-date-diff

feat: formula DATETIME_DIFF

packages/nc-gui/components/smartsheet/column/FormulaOptions.vue

@@ -235,6 +235,63 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
             },
             typeErrors,
           )
+        } else if (parsedTree.callee.name === 'DATETIME_DIFF') {
+          // parsedTree.arguments[0] = date
+          validateAgainstType(
+            parsedTree.arguments[0],
+            formulaTypes.DATE,
+            (v: any) => {
+              if (!validateDateWithUnknownFormat(v)) {
+                typeErrors.add('The first parameter of DATETIME_DIFF() should have date value')
+              }
+            },
+            typeErrors,
+          )
+          // parsedTree.arguments[1] = date
+          validateAgainstType(
+            parsedTree.arguments[1],
+            formulaTypes.DATE,
+            (v: any) => {
+              if (!validateDateWithUnknownFormat(v)) {
+                typeErrors.add('The second parameter of DATETIME_DIFF() should have date value')
+              }
+            },
+            typeErrors,
+          )
+          // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"]
+          validateAgainstType(
+            parsedTree.arguments[2],
+            formulaTypes.STRING,
+            (v: any) => {
+              if (
+                ![
+                  'milliseconds',
+                  'ms',
+                  'seconds',
+                  's',
+                  'minutes',
+                  'm',
+                  'hours',
+                  'h',
+                  'days',
+                  'd',
+                  'weeks',
+                  'w',
+                  'months',
+                  'M',
+                  'quarters',
+                  'Q',
+                  'years',
+                  'y',
+                ].includes(v)
+              ) {
+                typeErrors.add(
+                  'The third parameter of DATETIME_DIFF() should have value either "milliseconds", "ms", "seconds", "s", "minutes", "m", "hours", "h", "days", "d", "weeks", "w", "months", "M", "quarters", "Q", "years", or "y"',
+                )
+              }
+            },
+            typeErrors,
+          )
         }
       }
     }

packages/nc-gui/utils/formulaUtils.ts

@@ -51,6 +51,29 @@ const formulas: Record<string, any> = {
       'DATEADD({column1}, -2, "year")',
     ],
   },
+  DATETIME_DIFF: {
+    type: formulaTypes.DATE,
+    validation: {
+      args: {
+        min: 2,
+        max: 3,
+      },
+    },
+    description: 'Calculate the difference of two given date / datetime in specified units.',
+    syntax:
+      'DATETIME_DIFF(date | datetime, date | datetime, ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"])',
+    examples: [
+      'DATEDIFF({column1}, {column2})',
+      'DATEDIFF({column1}, {column2}, "seconds")',
+      'DATEDIFF({column1}, {column2}, "s")',
+      'DATEDIFF({column1}, {column2}, "years")',
+      'DATEDIFF({column1}, {column2}, "y")',
+      'DATEDIFF({column1}, {column2}, "minutes")',
+      'DATEDIFF({column1}, {column2}, "m")',
+      'DATEDIFF({column1}, {column2}, "days")',
+      'DATEDIFF({column1}, {column2}, "d")',
+    ],
+  },
   AND: {
     type: formulaTypes.COND_EXP,
     validation: {

packages/noco-docs/content/en/setup-and-usages/formulas.md

@@ -97,6 +97,8 @@ Example: ({Column1} + ({Column2} * {Column3}) / (3 - $Column4$ ))
 |  |  | `DATEADD(date, 1, 'year')` | Supposing {DATE_COL} is 2022-03-14 03:14. The result is 2023-03-14 03:14. | DateTime columns and negative values are supported. Example: `DATEADD(DATE_TIME_COL, -1, 'year')` |
 |  |  | `IF(NOW() < DATEADD(date,10,'day'), "true", "false")` | If the current date is less than {DATE_COL} plus 10 days, it returns true. Otherwise, it returns false. | DateTime columns and negative values are supported. |
 |  |  | `IF(NOW() < DATEADD(date,10,'day'), "true", "false")` | If the current date is less than {DATE_COL} plus 10 days, it returns true. Otherwise, it returns false. | DateTime columns and negative values are supported. |
+| **DATETIME_DIFF** | `DATETIME_DIFF(date, date, ["milliseconds" \| "ms" \| "seconds" \| "s" \| "minutes" \| "m" \| "hours" \| "h" \| "days" \| "d" \| "weeks" \| "w" \| "months" \| "M" \| "quarters" \| "Q" \| "years" \| "y"])` | `DATETIME_DIFF("2022/10/14", "2022/10/15", "second")` | Supposing {DATE_COL_1} is 2017-08-25 and {DATE_COL_2} is 2011-08-25. The result is 86400. | Compares two dates and returns the difference in the unit specified. Positive integers indicate the second date being in the past compared to the first and vice versa for negative ones. |
+|  |  | `WEEKDAY(NOW(), "sunday")` | If today is Monday, it returns 1 | Get the week day of NOW() with the first day set as sunday |
 | **WEEKDAY** | `WEEKDAY(date, [startDayOfWeek])` | `WEEKDAY(NOW())` | If today is Monday, it returns 0 | Returns the day of the week as an integer between 0 and 6 inclusive starting from Monday by default. You can optionally change the start day of the week by specifying in the second argument |
 |  |  | `WEEKDAY(NOW(), "sunday")` | If today is Monday, it returns 1 | Get the week day of NOW() with the first day set as sunday |
 

packages/nocodb-sdk/src/lib/formulaHelpers.ts

@@ -132,6 +132,7 @@ export function jsepTreeToFormula(node) {
       'AVG',
       'ADD',
       'DATEADD',
+      'DATETIME_DIFF',
       'WEEKDAY',
       'AND',
       'OR',

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mssql.ts

@@ -1,6 +1,7 @@
 import dayjs from 'dayjs';
 import { MapFnArgs } from '../mapFunctionName';
 import commonFns from './commonFns';
+import { convertUnits } from '../helpers/convertUnits';
 import { getWeekdayByText } from '../helpers/formulaFnHelper';
 
 const mssql = {
@@ -110,6 +111,17 @@ const mssql = {
       END${colAlias}`
     );
   },
+  DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => {
+    const datetime_expr1 = fn(pt.arguments[0]);
+    const datetime_expr2 = fn(pt.arguments[1]);
+    const rawUnit = pt.arguments[2]
+      ? fn(pt.arguments[2]).bindings[0]
+      : 'seconds';
+    const unit = convertUnits(rawUnit, 'mssql');
+    return knex.raw(
+      `DATEDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}`
+    );
+  },
   WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => {
     // DATEPART(WEEKDAY, DATE): sunday = 1, monday = 2, ..., saturday = 7
     // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/mysql.ts

@@ -1,6 +1,7 @@
 import dayjs from 'dayjs';
 import { MapFnArgs } from '../mapFunctionName';
 import commonFns from './commonFns';
+import { convertUnits } from '../helpers/convertUnits';
 import { getWeekdayByText } from '../helpers/formulaFnHelper';
 
 const mysql2 = {
@@ -61,6 +62,26 @@ const mysql2 = {
       END${colAlias}`
     );
   },
+  DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => {
+    const datetime_expr1 = fn(pt.arguments[0]);
+    const datetime_expr2 = fn(pt.arguments[1]);
+
+    const unit = convertUnits(
+      pt.arguments[2] ? fn(pt.arguments[2]).bindings[0] : 'seconds',
+      'mysql'
+    );
+
+    if (unit === 'MICROSECOND') {
+      // MySQL doesn't support millisecond
+      // hence change from MICROSECOND to millisecond manually
+      return knex.raw(
+        `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) div 1000 ${colAlias}`
+      );
+    }
+    return knex.raw(
+      `TIMESTAMPDIFF(${unit}, ${datetime_expr2}, ${datetime_expr1}) ${colAlias}`
+    );
+  },
   WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => {
     // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday
     return knex.raw(

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/pg.ts

@@ -1,6 +1,7 @@
 import dayjs from 'dayjs';
 import { MapFnArgs } from '../mapFunctionName';
 import commonFns from './commonFns';
+import { convertUnits } from '../helpers/convertUnits';
 import { getWeekdayByText } from '../helpers/formulaFnHelper';
 
 const pg = {
@@ -50,6 +51,56 @@ const pg = {
       )}')::interval${colAlias}`
     );
   },
+  DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => {
+    const datetime_expr1 = fn(pt.arguments[0]);
+    const datetime_expr2 = fn(pt.arguments[1]);
+    const rawUnit = pt.arguments[2]
+      ? fn(pt.arguments[2]).bindings[0]
+      : 'seconds';
+    let sql;
+    const unit = convertUnits(rawUnit, 'pg');
+    switch (unit) {
+      case 'second':
+        sql = `EXTRACT(EPOCH from (${datetime_expr1}::TIMESTAMP - ${datetime_expr2}::TIMESTAMP))::INTEGER`;
+        break;
+      case 'minute':
+        sql = `EXTRACT(EPOCH from (${datetime_expr1}::TIMESTAMP - ${datetime_expr2}::TIMESTAMP))::INTEGER / 60`;
+        break;
+      case 'milliseconds':
+        sql = `EXTRACT(EPOCH from (${datetime_expr1}::TIMESTAMP - ${datetime_expr2}::TIMESTAMP))::INTEGER * 1000`;
+        break;
+      case 'hour':
+        sql = `EXTRACT(EPOCH from (${datetime_expr1}::TIMESTAMP - ${datetime_expr2}::TIMESTAMP))::INTEGER / 3600`;
+        break;
+      case 'week':
+        sql = `TRUNC(DATE_PART('day', ${datetime_expr1}::TIMESTAMP - ${datetime_expr2}::TIMESTAMP) / 7)`;
+        break;
+      case 'month':
+        sql = `(
+                DATE_PART('year', ${datetime_expr1}::TIMESTAMP) - 
+                DATE_PART('year', ${datetime_expr2}::TIMESTAMP)
+               ) * 12 + (
+                DATE_PART('month', ${datetime_expr1}::TIMESTAMP) - 
+                DATE_PART('month', ${datetime_expr2}::TIMESTAMP)
+               )`;
+        break;
+      case 'quarter':
+        sql = `((EXTRACT(QUARTER FROM ${datetime_expr1}::TIMESTAMP) + 
+                    DATE_PART('year', AGE(${datetime_expr1}, '1900/01/01')) * 4) - 1) - 
+                ((EXTRACT(QUARTER FROM ${datetime_expr2}::TIMESTAMP) + 
+                    DATE_PART('year', AGE(${datetime_expr2}, '1900/01/01')) * 4) - 1)`;
+        break;
+      case 'year':
+        sql = `DATE_PART('year', ${datetime_expr1}::TIMESTAMP) - DATE_PART('year', ${datetime_expr2}::TIMESTAMP)`;
+        break;
+      case 'day':
+        sql = `DATE_PART('day', ${datetime_expr1}::TIMESTAMP - ${datetime_expr2}::TIMESTAMP)`;
+        break;
+      default:
+        sql = '';
+    }
+    return knex.raw(`${sql} ${colAlias}`);
+  },
   WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => {
     // isodow: the day of the week as Monday (1) to Sunday (7)
     // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/functionMappings/sqlite.ts

@@ -1,7 +1,12 @@
 import dayjs from 'dayjs';
 import { MapFnArgs } from '../mapFunctionName';
 import commonFns from './commonFns';
+import { convertUnits } from '../helpers/convertUnits';
 import { getWeekdayByText } from '../helpers/formulaFnHelper';
+import {
+  convertToTargetFormat,
+  getDateFormat,
+} from '../../../../../utils/dateTimeUtils';
 
 const sqlite3 = {
   ...commonFns,
@@ -77,7 +82,63 @@ const sqlite3 = {
       END${colAlias}`
     );
   },
-
+  DATETIME_DIFF: ({ fn, knex, pt, colAlias }: MapFnArgs) => {
+    let datetime_expr1 = fn(pt.arguments[0]).bindings[0];
+    let datetime_expr2 = fn(pt.arguments[1]).bindings[0];
+    // JULIANDAY takes YYYY-MM-DD
+    datetime_expr1 = convertToTargetFormat(
+      datetime_expr1,
+      getDateFormat(datetime_expr1),
+      'YYYY-MM-DD'
+    );
+    datetime_expr2 = convertToTargetFormat(
+      datetime_expr2,
+      getDateFormat(datetime_expr2),
+      'YYYY-MM-DD'
+    );
+    const rawUnit = pt.arguments[2]
+      ? fn(pt.arguments[2]).bindings[0]
+      : 'seconds';
+    let sql;
+    const unit = convertUnits(rawUnit, 'sqlite');
+    switch (unit) {
+      case 'seconds':
+        sql = `ROUND((JULIANDAY('${datetime_expr1}') - JULIANDAY('${datetime_expr2}')) * 86400)`;
+        break;
+      case 'minutes':
+        sql = `ROUND((JULIANDAY('${datetime_expr1}') - JULIANDAY('${datetime_expr2}')) * 1440)`;
+        break;
+      case 'hours':
+        sql = `ROUND((JULIANDAY('${datetime_expr1}') - JULIANDAY('${datetime_expr2}')) * 24)`;
+        break;
+      case 'milliseconds':
+        sql = `ROUND((JULIANDAY('${datetime_expr1}') - JULIANDAY('${datetime_expr2}')) * 86400000)`;
+        break;
+      case 'weeks':
+        sql = `ROUND((JULIANDAY('${datetime_expr1}') - JULIANDAY('${datetime_expr2}')) / 7)`;
+        break;
+      case 'months':
+        sql = `(ROUND((JULIANDAY('${datetime_expr1}') - JULIANDAY('${datetime_expr2}')) / 365))
+        * 12 + (ROUND((JULIANDAY('${datetime_expr1}') - JULIANDAY('${datetime_expr2}')) / 365 / 12))`;
+        break;
+      case 'quarters':
+        sql = `
+            ROUND((JULIANDAY('${datetime_expr1}')) / 365 / 4) -
+            ROUND((JULIANDAY('${datetime_expr2}')) / 365 / 4) +
+            (ROUND((JULIANDAY('${datetime_expr1}') - JULIANDAY('${datetime_expr2}')) / 365)) * 4
+        `;
+        break;
+      case 'years':
+        sql = `ROUND((JULIANDAY('${datetime_expr1}') - JULIANDAY('${datetime_expr2}')) / 365)`;
+        break;
+      case 'days':
+        sql = `JULIANDAY('${datetime_expr1}') - JULIANDAY('${datetime_expr2}')`;
+        break;
+      default:
+        sql = '';
+    }
+    return knex.raw(`${sql} ${colAlias}`);
+  },
   WEEKDAY: ({ fn, knex, pt, colAlias }: MapFnArgs) => {
     // strftime('%w', date) - day of week 0 - 6 with Sunday == 0
     // WEEKDAY() returns an index from 0 to 6 for Monday to Sunday

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/convertUnits.ts

@@ -0,0 +1,137 @@
+export function convertUnits(
+  unit: string,
+  type: 'mysql' | 'mssql' | 'pg' | 'sqlite'
+) {
+  switch (unit) {
+    case 'milliseconds':
+    case 'ms': {
+      switch (type) {
+        case 'mssql':
+          return 'millisecond';
+        case 'mysql':
+          // MySQL doesn't support millisecond
+          // hence change from MICROSECOND to millisecond manually
+          return 'MICROSECOND';
+        case 'pg':
+        case 'sqlite':
+          return 'milliseconds';
+        default:
+          return unit;
+      }
+    }
+    case 'seconds':
+    case 's': {
+      switch (type) {
+        case 'mssql':
+        case 'pg':
+          return 'second';
+        case 'mysql':
+          return 'SECOND';
+        case 'sqlite':
+          return 'seconds';
+        default:
+          return unit;
+      }
+    }
+    case 'minutes':
+    case 'm': {
+      switch (type) {
+        case 'mssql':
+        case 'pg':
+          return 'minute';
+        case 'mysql':
+          return 'MINUTE';
+        case 'sqlite':
+          return 'minutes';
+        default:
+          return unit;
+      }
+    }
+    case 'hours':
+    case 'h': {
+      switch (type) {
+        case 'mssql':
+        case 'pg':
+          return 'hour';
+        case 'mysql':
+          return 'HOUR';
+        case 'sqlite':
+          return 'hours';
+        default:
+          return unit;
+      }
+    }
+    case 'days':
+    case 'd': {
+      switch (type) {
+        case 'mssql':
+        case 'pg':
+          return 'day';
+        case 'mysql':
+          return 'DAY';
+        case 'sqlite':
+          return 'days';
+        default:
+          return unit;
+      }
+    }
+    case 'weeks':
+    case 'w': {
+      switch (type) {
+        case 'mssql':
+        case 'pg':
+          return 'week';
+        case 'mysql':
+          return 'WEEK';
+        case 'sqlite':
+          return 'weeks';
+        default:
+          return unit;
+      }
+    }
+    case 'months':
+    case 'M': {
+      switch (type) {
+        case 'mssql':
+        case 'pg':
+          return 'month';
+        case 'mysql':
+          return 'MONTH';
+        case 'sqlite':
+          return 'months';
+        default:
+          return unit;
+      }
+    }
+    case 'quarters':
+    case 'Q': {
+      switch (type) {
+        case 'mssql':
+        case 'pg':
+          return 'quarter';
+        case 'mysql':
+          return 'QUARTER';
+        case 'sqlite':
+          return 'quarters';
+        default:
+          return unit;
+      }
+    }
+    case 'years':
+    case 'y': {
+      switch (type) {
+        case 'mssql':
+        case 'pg':
+          return 'year';
+        case 'mysql':
+          return 'YEAR';
+        case 'sqlite':
+          return 'years';
+        default:
+          return unit;
+      }
+    }
+    default:
+      return unit;
+  }
+}

packages/nocodb/src/lib/utils/dateTimeUtils.ts

@@ -0,0 +1,75 @@
+import dayjs from 'dayjs';
+
+export const dateFormats = [
+  'DD-MM-YYYY',
+  'MM-DD-YYYY',
+  'YYYY-MM-DD',
+  'DD/MM/YYYY',
+  'MM/DD/YYYY',
+  'YYYY/MM/DD',
+  'DD MM YYYY',
+  'MM DD YYYY',
+  'YYYY MM DD',
+];
+
+export function validateDateFormat(v: string) {
+  return dateFormats.includes(v);
+}
+
+export function validateDateWithUnknownFormat(v: string) {
+  for (const format of dateFormats) {
+    if (dayjs(v, format, true).isValid() as any) {
+      return true;
+    }
+    for (const timeFormat of ['HH:mm', 'HH:mm:ss', 'HH:mm:ss.SSS']) {
+      if (dayjs(v, `${format} ${timeFormat}`, true).isValid() as any) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+export function getDateFormat(v: string) {
+  for (const format of dateFormats) {
+    if (dayjs(v, format, true).isValid()) {
+      return format;
+    }
+  }
+  return 'YYYY/MM/DD';
+}
+
+export function getDateTimeFormat(v: string) {
+  for (const format of dateFormats) {
+    for (const timeFormat of ['HH:mm', 'HH:mm:ss', 'HH:mm:ss.SSS']) {
+      const dateTimeFormat = `${format} ${timeFormat}`;
+      if (dayjs(v, dateTimeFormat, true).isValid() as any) {
+        return dateTimeFormat;
+      }
+    }
+  }
+  return 'YYYY/MM/DD';
+}
+
+export function parseStringDate(v: string, dateFormat: string) {
+  const dayjsObj = dayjs(v);
+  if (dayjsObj.isValid()) {
+    v = dayjsObj.format('YYYY-MM-DD');
+  } else {
+    v = dayjs(v, dateFormat).format('YYYY-MM-DD');
+  }
+  return v;
+}
+
+export function convertToTargetFormat(
+  v: string,
+  oldDataFormat,
+  newDateFormat: string
+) {
+  if (
+    !dateFormats.includes(oldDataFormat) ||
+    !dateFormats.includes(newDateFormat)
+  )
+    return v;
+  return dayjs(v, oldDataFormat).format(newDateFormat);
+}

tests/playwright/tests/columnFormula.spec.ts

@@ -30,6 +30,46 @@ const formulaDataByDbType = (context: NcContext) => [
     formula: `WEEKDAY("2022-07-19", "sunday")`,
     result: ['2', '2', '2', '2', '2'],
   },
+  {
+    formula: `DATETIME_DIFF("2022/10/14", "2022/10/15")`,
+    result: ['-86400', '-86400', '-86400', '-86400', '-86400'],
+  },
+  {
+    formula: `DATETIME_DIFF("2022/10/14", "2022/10/15", "minutes")`,
+    result: ['-1440', '-1440', '-1440', '-1440', '-1440'],
+  },
+  {
+    formula: `DATETIME_DIFF("2022/10/14", "2022/10/15", "seconds")`,
+    result: ['-86400', '-86400', '-86400', '-86400', '-86400'],
+  },
+  {
+    formula: `DATETIME_DIFF("2022/10/14", "2022/10/15", "milliseconds")`,
+    result: ['-86400000', '-86400000', '-86400000', '-86400000', '-86400000'],
+  },
+  {
+    formula: `DATETIME_DIFF("2022/10/14", "2022/10/15", "hours")`,
+    result: ['-24', '-24', '-24', '-24', '-24'],
+  },
+  {
+    formula: `DATETIME_DIFF("2022/10/14", "2023/10/14", "w")`,
+    result: ['-52', '-52', '-52', '-52', '-52'],
+  },
+  {
+    formula: `DATETIME_DIFF("2022/10/14", "2023/10/14", "M")`,
+    result: ['-12', '-12', '-12', '-12', '-12'],
+  },
+  {
+    formula: `DATETIME_DIFF("2022/10/14", "2023/10/14", "Q")`,
+    result: ['-4', '-4', '-4', '-4', '-4'],
+  },
+  {
+    formula: `DATETIME_DIFF("2022/10/14", "2023/10/14", "y")`,
+    result: ['-1', '-1', '-1', '-1', '-1'],
+  },
+  {
+    formula: `DATETIME_DIFF("2022/10/14", "2023/10/14", "d")`,
+    result: ['-365', '-365', '-365', '-365', '-365'],
+  },
   {
     formula: `CONCAT(UPPER({City}), LOWER({City}), TRIM('    trimmed    '))`,
     result: [