Merge pull request #9013 from nocodb/nc-fix/misc

Nc fix/misc

packages/nc-gui/components/smartsheet/column/FormulaOptions.vue

@@ -37,7 +37,7 @@ const supportedColumns = computed(
         return false
       }
 
-      if (isHiddenCol(col)) {
+      if (isHiddenCol(col, meta.value)) {
         return false
       }
 

packages/nc-gui/components/smartsheet/toolbar/CreateGroupBy.vue

@@ -32,7 +32,7 @@ const options = computed<ColumnType[]>(
             return false
           }
 
-          if (isHiddenCol(c)) {
+          if (isHiddenCol(c, meta.value)) {
             /** ignore mm relation column, created by and last modified by system field */
             return false
           }

packages/nc-gui/components/smartsheet/toolbar/CreateSort.vue

@@ -27,7 +27,7 @@ const options = computed<ColumnType[]>(
           return true
         }
         if (isSystemColumn(metaColumnById?.value?.[c.id!])) {
-          if (isHiddenCol(c)) {
+          if (isHiddenCol(c, meta.value)) {
             /** ignore mm relation column, created by and last modified by system field */
             return false
           }

packages/nc-gui/components/smartsheet/toolbar/FieldListAutoCompleteDropdown.vue

@@ -39,7 +39,7 @@ const options = computed<SelectProps['options']>(() =>
       }
 
       if (isSystemColumn(metaColumnById?.value?.[c.id!])) {
-        if (isHiddenCol(c)) {
+        if (isHiddenCol(c, meta.value)) {
           /** ignore mm relation column, created by and last modified by system field */
           return false
         }
@@ -60,7 +60,7 @@ const options = computed<SelectProps['options']>(() =>
         return true
       }
       if (isSystemColumn(metaColumnById?.value?.[c.id!])) {
-        if (isHiddenCol(c)) {
+        if (isHiddenCol(c, meta.value)) {
           /** ignore mm relation column, created by and last modified by system field */
           return false
         }

packages/nc-gui/components/smartsheet/toolbar/SearchData.vue

@@ -21,7 +21,11 @@ const globalSearchWrapperRef = ref<HTMLInputElement>()
 const { isMobileMode } = useGlobal()
 
 const columns = computed(
-  () => (meta.value as TableType)?.columns?.filter((column) => !isSystemColumn(column) && column?.uidt !== UITypes.Links) ?? [],
+  () =>
+    (meta.value as TableType)?.columns?.filter(
+      (column) =>
+        !isSystemColumn(column) && ![UITypes.Links, UITypes.Rollup, UITypes.DateTime, UITypes.Date].includes(column?.uidt),
+    ) ?? [],
 )
 
 watch(

packages/nc-gui/composables/useViewColumns.ts

@@ -87,7 +87,7 @@ const [useProvideViewColumns, useViewColumns] = useInjectionState(
         fields.value = meta.value?.columns
           ?.filter((column: ColumnType) => {
             // filter created by and last modified by system columns
-            if (isHiddenCol(column)) return false
+            if (isHiddenCol(column, meta.value)) return false
             return true
           })
           .map((column: ColumnType) => {

packages/nc-gui/composables/useViewGroupBy.ts

@@ -200,7 +200,7 @@ const [useProvideViewGroupBy, useViewGroupBy] = useInjectionState(
         } else if (
           [UITypes.Date, UITypes.DateTime, UITypes.CreatedTime, UITypes.LastModifiedTime].includes(curr.column_uidt as UITypes)
         ) {
-          acc += `${acc.length ? '~and' : ''}(${curr.title},eq,exactDate,${curr.key})`
+          acc += `${acc.length ? '~and' : ''}(${curr.title},gb_eq,exactDate,${curr.key})`
         } else if ([UITypes.User, UITypes.CreatedBy, UITypes.LastModifiedBy].includes(curr.column_uidt as UITypes)) {
           try {
             const value = JSON.parse(curr.key)

packages/nocodb-sdk/src/lib/UITypes.ts

@@ -1,5 +1,6 @@
-import { ColumnReqType, ColumnType } from './Api';
+import { ColumnReqType, ColumnType, TableType } from './Api';
 import { FormulaDataTypes } from './formulaHelpers';
+import { RelationTypes } from '~/lib/globals';
 
 enum UITypes {
   ID = 'ID',
@@ -208,17 +209,25 @@ export function isCreatedOrLastModifiedByCol(
 }
 
 export function isHiddenCol(
-  col: (ColumnReqType | ColumnType) & { system?: number | boolean }
+  col: (ColumnReqType | ColumnType) & {
+    colOptions?: any;
+    system?: number | boolean;
+  },
+  tableMeta: Partial<TableType>
 ) {
-  return (
-    col.system &&
-    (
-      [
-        UITypes.CreatedBy,
-        UITypes.LastModifiedBy,
-        UITypes.LinkToAnotherRecord,
-      ] as string[]
-    ).includes(col.uidt)
+  if (!col.system) return false;
+
+  // hide belongs to column in mm tables only
+  if (col.uidt === UITypes.LinkToAnotherRecord) {
+    if (col.colOptions?.type === RelationTypes.BELONGS_TO && tableMeta?.mm) {
+      return true;
+    }
+    // hide system columns in other tables which are has-many used for mm
+    return col.colOptions?.type === RelationTypes.HAS_MANY;
+  }
+
+  return ([UITypes.CreatedBy, UITypes.LastModifiedBy] as string[]).includes(
+    col.uidt
   );
 }
 

packages/nocodb/src/db/BaseModelSqlv2.ts

@@ -1154,6 +1154,64 @@ class BaseModelSqlv2 {
               groupBySelectors.push(column.id);
             }
             break;
+          case UITypes.CreatedTime:
+          case UITypes.LastModifiedTime:
+          case UITypes.DateTime:
+            {
+              const columnName = await getColumnName(
+                this.context,
+                column,
+                columns,
+              );
+              // ignore seconds part in datetime and group
+              if (this.dbDriver.clientType() === 'pg') {
+                selectors.push(
+                  this.dbDriver.raw(
+                    "date_trunc('minute', ??) + interval '0 seconds' as ??",
+                    [columnName, column.id],
+                  ),
+                );
+              } else if (
+                this.dbDriver.clientType() === 'mysql' ||
+                this.dbDriver.clientType() === 'mysql2'
+              ) {
+                selectors.push(
+                  // this.dbDriver.raw('??::date as ??', [columnName, column.id]),
+                  this.dbDriver.raw(
+                    "DATE_SUB(CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00'), INTERVAL SECOND(??) SECOND) as ??",
+                    [columnName, columnName, column.id],
+                  ),
+                );
+              } else if (this.dbDriver.clientType() === 'sqlite3') {
+                selectors.push(
+                  this.dbDriver.raw(
+                    `strftime ('%Y-%m-%d %H:%M:00',:column:) ||
+  (
+   CASE WHEN substr(:column:, 20, 1) = '+' THEN
+    printf ('+%s:',
+     substr(:column:, 21, 2)) || printf ('%s',
+     substr(:column:, 24, 2))
+   WHEN substr(:column:, 20, 1) = '-' THEN
+    printf ('-%s:',
+     substr(:column:, 21, 2)) || printf ('%s',
+     substr(:column:, 24, 2))
+   ELSE
+    '+00:00'
+   END) AS :id:`,
+                    {
+                      column: columnName,
+                      id: column.id,
+                    },
+                  ),
+                );
+              } else {
+                selectors.push(
+                  this.dbDriver.raw('DATE(??) as ??', [columnName, column.id]),
+                );
+              }
+              groupBySelectors.push(column.id);
+            }
+            break;
           default:
             {
               const columnName = await getColumnName(
@@ -1396,6 +1454,63 @@ class BaseModelSqlv2 {
               groupBySelectors.push(column.id);
             }
             break;
+          case UITypes.CreatedTime:
+          case UITypes.LastModifiedTime:
+          case UITypes.DateTime:
+            {
+              const columnName = await getColumnName(
+                this.context,
+                column,
+                columns,
+              );
+              // ignore seconds part in datetime and group
+              if (this.dbDriver.clientType() === 'pg') {
+                selectors.push(
+                  this.dbDriver.raw(
+                    "date_trunc('minute', ??) + interval '0 seconds' as ??",
+                    [columnName, column.id],
+                  ),
+                );
+              } else if (
+                this.dbDriver.clientType() === 'mysql' ||
+                this.dbDriver.clientType() === 'mysql2'
+              ) {
+                selectors.push(
+                  this.dbDriver.raw(
+                    "CONVERT_TZ(DATE_SUB(??, INTERVAL SECOND(??) SECOND), @@GLOBAL.time_zone, '+00:00') as ??",
+                    [columnName, columnName, column.id],
+                  ),
+                );
+              } else if (this.dbDriver.clientType() === 'sqlite3') {
+                selectors.push(
+                  this.dbDriver.raw(
+                    `strftime ('%Y-%m-%d %H:%M:00',:column:) ||
+  (
+   CASE WHEN substr(:column:, 20, 1) = '+' THEN
+    printf ('+%s:',
+     substr(:column:, 21, 2)) || printf ('%s',
+     substr(:column:, 24, 2))
+   WHEN substr(:column:, 20, 1) = '-' THEN
+    printf ('-%s:',
+     substr(:column:, 21, 2)) || printf ('%s',
+     substr(:column:, 24, 2))
+   ELSE
+    '+00:00'
+   END) as :id:`,
+                    {
+                      column: columnName,
+                      id: column.id,
+                    },
+                  ),
+                );
+              } else {
+                selectors.push(
+                  this.dbDriver.raw('DATE(??) as ??', [columnName, column.id]),
+                );
+              }
+              groupBySelectors.push(column.id);
+            }
+            break;
           default:
             {
               const columnName = await getColumnName(

packages/nocodb/src/db/conditionV2.ts

@@ -76,7 +76,7 @@ const parseConditionV2 = async (
 
   const context = baseModelSqlv2.context;
 
-  let filter: Filter;
+  let filter: Filter & { groupby?: boolean };
   if (!Array.isArray(_filter)) {
     if (!(_filter instanceof Filter)) filter = new Filter(_filter as Filter);
     else filter = _filter;
@@ -136,7 +136,7 @@ const parseConditionV2 = async (
       (filter.comparison_op as any) === 'gb_eq' ||
       (filter.comparison_op as any) === 'gb_null'
     ) {
-      (filter as any).groupby = true;
+      filter.groupby = true;
 
       const column = await getRefColumnIfAlias(
         context,
@@ -539,7 +539,7 @@ const parseConditionV2 = async (
             (val + '').startsWith('%') || (val + '').endsWith('%')
               ? val
               : `%${val}%`;
-          if (qb?.client?.config?.client === 'pg') {
+          if (knex.clientType() === 'pg') {
             qb = qb.where(knex.raw(`(${finalStatement}) ilike ?`, [val]));
           } else {
             qb = qb.where(knex.raw(`(${finalStatement}) like ?`, [val]));
@@ -553,7 +553,7 @@ const parseConditionV2 = async (
             val = val.startsWith('%') || val.endsWith('%') ? val : `%${val}%`;
 
             qb.where((nestedQb) => {
-              if (qb?.client?.config?.client === 'pg') {
+              if (knex.clientType() === 'pg') {
                 nestedQb.whereNot(
                   knex.raw(`(${finalStatement}) ilike ?`, [val]),
                 );
@@ -599,7 +599,7 @@ const parseConditionV2 = async (
         // todo: refactor this to use a better approach to make it more readable and clean
         let genVal = customWhereClause ? field : val;
         const dateFormat =
-          qb?.client?.config?.client === 'mysql2'
+          knex.clientType() === 'mysql2'
             ? 'YYYY-MM-DD HH:mm:ss'
             : 'YYYY-MM-DD HH:mm:ssZ';
 
@@ -712,7 +712,10 @@ const parseConditionV2 = async (
 
         switch (filter.comparison_op) {
           case 'eq':
-            if (qb?.client?.config?.client === 'mysql2') {
+            if (
+              knex.clientType() === 'mysql2' ||
+              knex.clientType() === 'mysql'
+            ) {
               if (
                 [
                   UITypes.Duration,
@@ -734,7 +737,17 @@ const parseConditionV2 = async (
                 column.ct === 'date' ||
                 column.ct === 'datetime'
               ) {
-                qb = qb.where(knex.raw('DATE(??) = DATE(?)', [field, val]));
+                // ignore seconds part in datetime and filter when using it for group by
+                if (filter.groupby && column.ct !== 'date') {
+                  const valWithoutTz = val.replace(/[+-]\d+:\d+$/, '');
+                  qb = qb.where(
+                    knex.raw(
+                      "CONVERT_TZ(DATE_SUB(??, INTERVAL SECOND(??) SECOND), @@GLOBAL.time_zone, '+00:00') = DATE_SUB(?, INTERVAL SECOND(?) SECOND)",
+                      [field, field, valWithoutTz, valWithoutTz],
+                    ),
+                  );
+                } else
+                  qb = qb.where(knex.raw('DATE(??) = DATE(?)', [field, val]));
               } else {
                 // mysql is case-insensitive for strings, turn to case-sensitive
                 qb = qb.where(knex.raw('BINARY ?? = ?', [field, val]));
@@ -751,13 +764,40 @@ const parseConditionV2 = async (
                 ].includes(column.uidt)
               ) {
                 if (qb.client.config.client === 'pg') {
-                  //  todo: enable back if group by date required custom implementation
-                  // if ((filter as any).groupby)
-                  //   qb = qb.where(knex.raw('??::timestamp = ?', [field, val]));
-                  // else
-                  qb = qb.where(knex.raw('??::date = ?', [field, val]));
+                  // ignore seconds part in datetime and filter when using it for group by
+                  if (filter.groupby)
+                    qb = qb.where(
+                      knex.raw(
+                        "date_trunc('minute', ??) + interval '0 seconds' = ?",
+                        [field, val],
+                      ),
+                    );
+                  else qb = qb.where(knex.raw('??::date = ?', [field, val]));
                 } else {
-                  qb = qb.where(knex.raw('DATE(??) = DATE(?)', [field, val]));
+                  // ignore seconds part in datetime and filter when using it for group by
+                  if (filter.groupby) {
+                    if (knex.clientType() === 'sqlite3')
+                      qb = qb.where(
+                        knex.raw(
+                          `Datetime(strftime ('%Y-%m-%d %H:%M:00',:column:) ||
+  (
+   CASE WHEN substr(:column:, 20, 1) = '+' THEN
+    printf ('+%s:',
+     substr(:column:, 21, 2)) || printf ('%s',
+     substr(:column:, 24, 2))
+   WHEN substr(:column:, 20, 1) = '-' THEN
+    printf ('-%s:',
+     substr(:column:, 21, 2)) || printf ('%s',
+     substr(:column:, 24, 2))
+   ELSE
+    '+00:00'
+   END)) = Datetime(:val)`,
+                          { column: field, val },
+                        ),
+                      );
+                    else qb = qb.where(knex.raw('?? = ?', [field, val]));
+                  } else
+                    qb = qb.where(knex.raw('DATE(??) = DATE(?)', [field, val]));
                 }
               } else {
                 qb = qb.where(field, val);
@@ -770,7 +810,7 @@ const parseConditionV2 = async (
             break;
           case 'neq':
           case 'not':
-            if (qb?.client?.config?.client === 'mysql2') {
+            if (knex.clientType() === 'mysql2') {
               if (
                 [
                   UITypes.Duration,
@@ -835,7 +875,7 @@ const parseConditionV2 = async (
                     ? val
                     : `%${val}%`;
               }
-              if (qb?.client?.config?.client === 'pg') {
+              if (knex.clientType() === 'pg') {
                 qb = qb.where(knex.raw('??::text ilike ?', [field, val]));
               } else {
                 qb = qb.where(field, 'like', val);
@@ -862,7 +902,7 @@ const parseConditionV2 = async (
                   val.startsWith('%') || val.endsWith('%') ? val : `%${val}%`;
               }
               qb.where((nestedQb) => {
-                if (qb?.client?.config?.client === 'pg') {
+                if (knex.clientType() === 'pg') {
                   nestedQb.where(
                     knex.raw('??::text not ilike ?', [field, val]),
                   );
@@ -891,9 +931,9 @@ const parseConditionV2 = async (
                 for (let i = 0; i < items?.length; i++) {
                   let sql;
                   const bindings = [field, `%,${items[i]},%`];
-                  if (qb?.client?.config?.client === 'pg') {
+                  if (knex.clientType() === 'pg') {
                     sql = "(',' || ??::text || ',') ilike ?";
-                  } else if (qb?.client?.config?.client === 'sqlite3') {
+                  } else if (knex.clientType() === 'sqlite3') {
                     sql = "(',' || ?? || ',') like ?";
                   } else {
                     sql = "CONCAT(',', ??, ',') like ?";

packages/nocodb/src/helpers/dataHelpers.ts

@@ -166,10 +166,9 @@ export async function serializeCellValue(
           data = JSON.parse(value);
         }
 
-        if(!Array.isArray(data)) {
+        if (!Array.isArray(data)) {
           data = undefined;
         }
-
       } catch {
         data = undefined;
       }

packages/nocodb/src/services/app-hooks-listener.service.ts

@@ -264,6 +264,14 @@ export class AppHooksListenerService implements OnModuleInit, OnModuleDestroy {
           });
         }
         break;
+      case AppEvents.ATTACHMENT_UPLOAD:
+        {
+          this.telemetryService.sendEvent({
+            evt_type: 'image:uploaded',
+            type: data?.type,
+          });
+        }
+        break;
     }
   }