Merge pull request #5678 from nocodb/fix/mssql-datetime

fix: datetime followup

packages/nc-gui/components/cell/DateTimePicker.vue

@@ -105,10 +105,9 @@ let localState = $computed({
     }
 
     if (val.isValid()) {
-      const formattedValue = dayjs(val?.format(isMysql(column.value.base_id) ? 'YYYY-MM-DD HH:mm:ss' : 'YYYY-MM-DD HH:mm:ssZ'))
       // setting localModelValue to cater NOW function in date picker
-      localModelValue = formattedValue
+      localModelValue = dayjs(val)
-      emit('update:modelValue', formattedValue)
+      emit('update:modelValue', dayjs(val).format(isMysql(column.value.base_id) ? 'YYYY-MM-DD HH:mm:ss' : 'YYYY-MM-DD HH:mm:ssZ'))
     }
   },
 })

packages/nc-gui/composables/useMultiSelect/index.ts

@@ -83,11 +83,19 @@ export function useMultiSelect(
   }
 
   function constructDateTimeFormat(column: ColumnType) {
-    const dateFormat = parseProp(column?.meta)?.date_format ?? dateFormats[0]
+    const dateFormat = constructDateFormat(column)
-    const timeFormat = parseProp(column?.meta)?.time_format ?? timeFormats[0]
+    const timeFormat = constructTimeFormat(column)
     return `${dateFormat} ${timeFormat}`
   }
 
+  function constructDateFormat(column: ColumnType) {
+    return parseProp(column?.meta)?.date_format ?? dateFormats[0]
+  }
+
+  function constructTimeFormat(column: ColumnType) {
+    return parseProp(column?.meta)?.time_format ?? timeFormats[0]
+  }
+
   async function copyValue(ctx?: Cell) {
     try {
       if (selectedRange.start !== null && selectedRange.end !== null && !selectedRange.isSingleCell()) {
@@ -124,7 +132,7 @@ export function useMultiSelect(
             })
           }
 
-          if (columnObj.uidt === UITypes.DateTime) {
+          if (columnObj.uidt === UITypes.DateTime || columnObj.uidt === UITypes.Time) {
             // remove `"`
             // e.g. "2023-05-12T08:03:53.000Z" -> 2023-05-12T08:03:53.000Z
             textToCopy = textToCopy.replace(/["']/g, '')
@@ -143,10 +151,12 @@ export function useMultiSelect(
             // users can change the datetime format in UI
             // `textToCopy` would be always in YYYY-MM-DD HH:mm:ss(Z / +xx:yy) format
             // therefore, here we reformat to the correct datetime format based on the meta
-            textToCopy = d.format(constructDateTimeFormat(columnObj))
+            textToCopy = d.format(
+              columnObj.uidt === UITypes.DateTime ? constructDateTimeFormat(columnObj) : constructTimeFormat(columnObj),
+            )
 
-            if (!dayjs(textToCopy).isValid()) {
+            if (columnObj.uidt === UITypes.DateTime && !dayjs(textToCopy).isValid()) {
-              throw new Error('Invalid Date')
+              throw new Error('Invalid DateTime')
             }
           }
 

packages/nc-gui/utils/cell.ts

@@ -61,21 +61,18 @@ export const renderValue = (result?: any) => {
     return result
   }
 
-  // cater MYSQL
-  result = result.replace('.000000', '')
-
   // convert ISO string (e.g. in MSSQL) to YYYY-MM-DD hh:mm:ssZ
   // e.g. 2023-05-18T05:30:00.000Z -> 2023-05-18 11:00:00+05:30
-  result = result.replace(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/, (d: string) => {
+  result = result.replace(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g, (d: string) => {
-    return dayjs(d).format('YYYY-MM-DD HH:mm:ssZ')
+    return dayjs(d).isValid() ? dayjs(d).format('YYYY-MM-DD HH:mm:ssZ') : d
   })
 
   // convert all date time values to local time
   // the datetime is either YYYY-MM-DD hh:mm:ss (xcdb)
-  // or YYYY-MM-DD hh:mm:ss+xx:yy (ext)
+  // or YYYY-MM-DD hh:mm:ss+/-xx:yy (ext)
-  return result.replace(/\b(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})(\[+-]\d{2}:\d{2})?\b/g, (d: string) => {
+  return result.replace(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?/g, (d: string) => {
     // TODO(timezone): retrieve the format from the corresponding column meta
     // assume hh:mm at this moment
-    return dayjs(d).utc(!result.includes('+')).local().format('YYYY-MM-DD HH:mm')
+    return dayjs(d).isValid() ? dayjs(d).format('YYYY-MM-DD HH:mm') : d
   })
 }

packages/nocodb/src/db/BaseModelSqlv2.ts

@@ -1652,15 +1652,17 @@ class BaseModelSqlv2 {
             }
           } else if (this.isMssql) {
             // if there is no timezone info, convert to database timezone, then convert to UTC
-            if (column.dt !== 'datetime2') {
+            if (column.dt !== 'datetimeoffset') {
-              const col = `${sanitize(alias || this.model.table_name)}.${
-                column.column_name
-              }`;
               res[sanitize(column.title || column.column_name)] =
                 this.dbDriver.raw(
-                  `SWITCHOFFSET(??, DATEPART(TZOFFSET, ??)) AT TIME ZONE 'UTC'`,
+                  `CONVERT(DATETIMEOFFSET, ?? AT TIME ZONE 'UTC')`,
-                  [col, col],
+                  [
+                    `${sanitize(alias || this.model.table_name)}.${
+                      column.column_name
+                    }`,
+                  ],
                 );
+              break;
             }
           }
           res[sanitize(column.title || column.column_name)] = sanitize(
@@ -3296,13 +3298,14 @@ class BaseModelSqlv2 {
             this._convertAttachmentType(attachmentColumns, d),
           );
         } else {
-          this._convertAttachmentType(attachmentColumns, data);
+          data = this._convertAttachmentType(attachmentColumns, data);
         }
       }
     }
     return data;
   }
 
+  // TODO(timezone): retrieve the format from the corresponding column meta
   private _convertDateFormat(
     dateTimeColumns: Record<string, any>[],
     d: Record<string, any>,
@@ -3311,6 +3314,60 @@ class BaseModelSqlv2 {
     for (const col of dateTimeColumns) {
       if (!d[col.title]) continue;
 
+      if (col.uidt === UITypes.Formula) {
+        if (!d[col.title] || typeof d[col.title] !== 'string') {
+          continue;
+        }
+
+        // cater MYSQL
+        d[col.title] = d[col.title].replace(/\.000000/g, '');
+
+        if (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g.test(d[col.title])) {
+          // convert ISO string (e.g. in MSSQL) to YYYY-MM-DD hh:mm:ssZ
+          // e.g. 2023-05-18T05:30:00.000Z -> 2023-05-18 11:00:00+05:30
+          d[col.title] = d[col.title].replace(
+            /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g,
+            (d: string) => {
+              return dayjs(d).isValid()
+                ? dayjs(d).utc(true).format('YYYY-MM-DD HH:mm:ssZ')
+                : d;
+            },
+          );
+          continue;
+        }
+
+        // convert all date time values to utc
+        // the datetime is either YYYY-MM-DD hh:mm:ss (xcdb)
+        // or YYYY-MM-DD hh:mm:ss+/-xx:yy (ext)
+        d[col.title] = d[col.title].replace(
+          /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?/g,
+          (d: string) => {
+            if (!dayjs(d).isValid()) {
+              return d;
+            }
+
+            if (this.isSqlite) {
+              // if there is no timezone info,
+              // we assume the input is on NocoDB server timezone
+              // then we convert to UTC from server timezone
+              // e.g. 2023-04-27 10:00:00 (IST) -> 2023-04-27 04:30:00+00:00
+              return dayjs(d)
+                .tz(Intl.DateTimeFormat().resolvedOptions().timeZone)
+                .utc()
+                .format('YYYY-MM-DD HH:mm:ssZ');
+            }
+
+            // set keepLocalTime to true if timezone info is not found
+            const keepLocalTime = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/g.test(
+              d,
+            );
+
+            return dayjs(d).utc(keepLocalTime).format('YYYY-MM-DD HH:mm:ssZ');
+          },
+        );
+        continue;
+      }
+
       let keepLocalTime = true;
 
       if (this.isSqlite) {
@@ -3367,12 +3424,14 @@ class BaseModelSqlv2 {
     if (data) {
       const dateTimeColumns = (
         childTable ? childTable.columns : this.model.columns
-      ).filter((c) => c.uidt === UITypes.DateTime);
+      ).filter(
+        (c) => c.uidt === UITypes.DateTime || c.uidt === UITypes.Formula,
+      );
       if (dateTimeColumns.length) {
         if (Array.isArray(data)) {
           data = data.map((d) => this._convertDateFormat(dateTimeColumns, d));
         } else {
-          this._convertDateFormat(dateTimeColumns, data);
+          data = this._convertDateFormat(dateTimeColumns, data);
         }
       }
     }

packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts

@@ -562,6 +562,19 @@ async function _formulaQueryBuilder(
                 .wrap('(', ')'),
             };
           };
+        } else if (
+          knex.clientType() === 'mssql' &&
+          col.dt !== 'datetimeoffset'
+        ) {
+          // if there is no timezone info, convert to database timezone, then convert to UTC
+          aliasToColumn[col.id] = async (): Promise<any> => {
+            return {
+              builder: knex.raw(
+                `CONVERT(DATETIMEOFFSET, ?? AT TIME ZONE 'UTC')`,
+                [col.column_name],
+              ),
+            };
+          };
         } else {
           aliasToColumn[col.id] = () =>
             Promise.resolve({ builder: col.column_name });

packages/nocodb/src/db/functionMappings/mssql.ts

@@ -130,15 +130,15 @@ const mssql = {
         )}, 
         ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${
           (await fn(pt.arguments[0])).builder
-        }), 'yyyy-MM-dd HH:mm')
+        }), 'yyyy-MM-dd HH:mm:ss')
       ELSE
        FORMAT(DATEADD(${String((await fn(pt.arguments[2])).builder).replace(
          /["']/g,
          '',
        )}, 
-       ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${fn(
+       ${dateIN > 0 ? '+' : ''}${(await fn(pt.arguments[1])).builder}, ${
-          pt.arguments[0],
+          (await fn(pt.arguments[0])).builder
-        )}), 'yyyy-MM-dd')
+        }), 'yyyy-MM-dd')
       END${colAlias}`,
       ),
     };

packages/nocodb/src/models/Model.ts

@@ -508,7 +508,7 @@ export default class Model implements TableType {
             // e.g. 2023-05-10T08:49:32.000Z -> 2023-05-10 08:49:32-08:00
             // then convert to db timezone
             val = knex.raw(
-              `SWITCHOFFSET(? AT TIME ZONE 'UTC', SYSDATETIMEOFFSET()`,
+              `SWITCHOFFSET(CONVERT(datetimeoffset, ?), DATENAME(TzOffset, SYSDATETIMEOFFSET()))`,
               [dayjs(val).utc().format('YYYY-MM-DD HH:mm:ssZ')],
             );
           } else {