refactor: bulk meta insert

packages/nocodb/src/cache/RedisCacheMgr.ts

@@ -23,7 +23,7 @@ export default class RedisCacheMgr extends CacheMgr {
       process.env.NC_CLOUD !== 'true'
     ) {
       // flush the existing db with selected key (Default: 0)
-      this.client.flushdb();
+      // this.client.flushdb();
     }
 
     // TODO(cache): fetch orgs once it's implemented

packages/nocodb/src/meta/meta.service.ts

@@ -118,26 +118,31 @@ export class MetaService {
     });
     return insertObj;
   }
-
   public async bulkMetaInsert(
     base_id: string,
     source_id: string,
     target: string,
-    data: any[],
+    data: any | any[],
     ignoreIdGeneration?: boolean,
   ): Promise<any> {
     const insertObj = [];
     const at = this.now();
-    for (const d of data) {
+
+    const commonProps: Record<string, any> = {
+      created_at: at,
+      updated_at: at,
+    };
+
+    if (source_id !== null) commonProps.source_id = source_id;
+    if (base_id !== null) commonProps.base_id = base_id;
+
+    for (const d of Array.isArray(data) ? data : [data]) {
       const id = d?.id || (await this.genNanoid(target));
       const tempObj = {
         ...d,
         ...(ignoreIdGeneration ? {} : { id }),
-        created_at: at,
-        updated_at: at,
+        ...commonProps,
       };
-      if (source_id !== null) tempObj.source_id = source_id;
-      if (base_id !== null) tempObj.base_id = base_id;
       insertObj.push(tempObj);
     }
 

packages/nocodb/src/models/Column.ts

@@ -1330,4 +1330,230 @@ export default class Column<T = any> implements ColumnType {
       colId,
     );
   }
+
+  static async bulkInsert(
+    param: {
+      columns: Column[];
+      fk_model_id: any;
+      source_id: string;
+      base_id: string;
+    },
+    ncMeta: MetaService = Noco.ncMeta,
+  ) {
+    const extractedColumnMetas = [];
+
+    // add fk_model_id
+    for (const column of param.columns) {
+      extractedColumnMetas.push({
+        ...extractProps(column as any, [
+          'id',
+          'fk_model_id',
+          'column_name',
+          'title',
+          'uidt',
+          'dt',
+          'np',
+          'ns',
+          'clen',
+          'cop',
+          'pk',
+          'rqd',
+          'un',
+          'ct',
+          'ai',
+          'unique',
+          'cdf',
+          'cc',
+          'csn',
+          'dtx',
+          'dtxp',
+          'dtxs',
+          'au',
+          'pv',
+          'order',
+          'base_id',
+          'source_id',
+          'system',
+          'meta',
+        ]),
+        fk_model_id: param.fk_model_id,
+      });
+    }
+
+    // bulk insert columns
+    const columns = await ncMeta.bulkMetaInsert(
+      null,
+      null,
+      MetaTable.COLUMNS,
+      extractedColumnMetas,
+    );
+
+    // insert column options if any
+    // for (const column of columns) {
+    await Column.bulkInsertColOption(columns, ncMeta);
+    // }
+
+    return columns;
+  }
+
+  private static async bulkInsertColOption<T>(
+    columns: (Partial<T> & { source_id?: string; [p: string]: any })[],
+    ncMeta = Noco.ncMeta,
+  ) {
+    const insertGroups = new Map<UITypes, Record<string, any>[]>();
+
+    for (const column of columns) {
+      let insertArr = insertGroups.get(column.uidt=== UITypes.MultiSelect ? UITypes.SingleSelect : column.uidt );
+      if (!insertArr) {
+        insertGroups.set(column.uidt, (insertArr = []));
+      }
+      switch (column.uidt || column.ui_data_type) {
+        case UITypes.Lookup:
+          // LookupColumn.insert()
+          insertArr.push({
+            fk_column_id: column.id,
+            fk_relation_column_id: column.fk_relation_column_id,
+            fk_lookup_column_id: column.fk_lookup_column_id,
+          });
+          break;
+
+        case UITypes.Rollup: {
+          insertArr.push({
+            fk_column_id: column.id,
+            fk_relation_column_id: column.fk_relation_column_id,
+
+            fk_rollup_column_id: column.fk_rollup_column_id,
+            rollup_function: column.rollup_function,
+          });
+          break;
+        }
+        case UITypes.Links:
+        case UITypes.LinkToAnotherRecord: {
+          insertArr.push({
+            fk_column_id: column.id,
+
+            // ref_db_alias
+            type: column.type,
+            // db_type:
+
+            fk_child_column_id: column.fk_child_column_id,
+            fk_parent_column_id: column.fk_parent_column_id,
+
+            fk_mm_model_id: column.fk_mm_model_id,
+            fk_mm_child_column_id: column.fk_mm_child_column_id,
+            fk_mm_parent_column_id: column.fk_mm_parent_column_id,
+
+            ur: column.ur,
+            dr: column.dr,
+
+            fk_index_name: column.fk_index_name,
+            fk_related_model_id: column.fk_related_model_id,
+
+            virtual: column.virtual,
+          });
+          break;
+        }
+        case UITypes.QrCode: {
+          insertArr.push(
+            {
+              fk_column_id: column.id,
+              fk_qr_value_column_id: column.fk_qr_value_column_id,
+            },
+            ncMeta,
+          );
+          break;
+        }
+        case UITypes.Barcode: {
+          insertArr.push({
+            fk_column_id: column.id,
+            fk_barcode_value_column_id: column.fk_barcode_value_column_id,
+            barcode_format: column.barcode_format,
+          });
+          break;
+        }
+        case UITypes.Formula: {
+          insertArr.push({
+            fk_column_id: column.id,
+            formula: column.formula,
+            formula_raw: column.formula_raw,
+            parsed_tree: column.parsed_tree,
+          });
+          break;
+        }
+        case UITypes.MultiSelect: {
+          if (!column.colOptions?.options) {
+            for (const [i, option] of column.dtxp?.split(',').entries() ||
+              [].entries()) {
+              insertArr.push({
+                fk_column_id: column.id,
+                title: option.replace(/^'/, '').replace(/'$/, ''),
+                order: i + 1,
+                color: selectColors[i % selectColors.length],
+              });
+            }
+          } else {
+            for (const [i, option] of column.colOptions.options.entries() ||
+              [].entries()) {
+              // Trim end of enum/set
+              if (column.dt === 'enum' || column.dt === 'set') {
+                option.title = option.title.trimEnd();
+              }
+              insertArr.push({
+                color: selectColors[i % selectColors.length], // in case color is not provided
+                ...option,
+                fk_column_id: column.id,
+                order: i + 1,
+              });
+            }
+          }
+          break;
+        }
+        case UITypes.SingleSelect: {
+          if (!column.colOptions?.options) {
+            for (const [i, option] of column.dtxp?.split(',').entries() ||
+              [].entries()) {
+              insertArr.push({
+                fk_column_id: column.id,
+                title: option.replace(/^'/, '').replace(/'$/, ''),
+                order: i + 1,
+                color: selectColors[i % selectColors.length],
+              });
+            }
+          } else {
+            for (const [i, option] of column.colOptions.options.entries() ||
+              [].entries()) {
+              // Trim end of enum/set
+              if (column.dt === 'enum' || column.dt === 'set') {
+                option.title = option.title.trimEnd();
+              }
+              insertArr.push({
+                color: selectColors[i % selectColors.length], // in case color is not provided
+                ...option,
+                fk_column_id: column.id,
+                order: i + 1,
+              });
+            }
+          }
+          break;
+        }
+      }
+    }
+
+    // bulk insert column options
+    for (const group of insertGroups.keys()) {
+      switch (group) {
+        case UITypes.SingleSelect:
+        case UITypes.MultiSelect:
+          await ncMeta.bulkMetaInsert(
+            null,
+            null,
+            MetaTable.COL_SELECT_OPTIONS,
+            insertGroups.get(group),
+          );
+          break;
+
+        // todo: handle rest of the cases
+      }
+    }
+  }
 }

packages/nocodb/src/models/Model.ts

@@ -147,21 +147,34 @@ export default class Model implements TableType {
       insertObj,
     );
 
-    const view = await View.insert(
+    const insertedColumns = await Column.bulkInsert(
+      {
+        columns: (model?.columns || []) as Column[],
+        fk_model_id: id,
+        source_id: sourceId,
+        base_id: baseId,
+      },
+      ncMeta,
+    );
+
+    await View.insertMetaOnly(
       {
         fk_model_id: id,
         title: model.title || model.table_name,
         is_default: true,
         type: ViewTypes.GRID,
+        base_id: baseId,
+        source_id: sourceId,
+      },
+      {
+        getColumns: () => insertedColumns,
       },
       ncMeta,
     );
 
-    for (const column of model?.columns || []) {
-      await Column.insert({ ...column, fk_model_id: id, view } as any, ncMeta);
-    }
+    const model = await this.getWithInfo({ id }, ncMeta);
 
-    return this.getWithInfo({ id }, ncMeta).then(async (model) => {
+    // append to model list since model list cache will be there already
     if (sourceId) {
       await NocoCache.appendToList(
         CacheScope.MODEL,
@@ -176,8 +189,8 @@ export default class Model implements TableType {
       [baseId],
       `${CacheScope.MODEL}:${id}`,
     );
+
     return model;
-    });
   }
 
   public static async list(

packages/nocodb/src/models/View.ts

@@ -1539,4 +1539,271 @@ export default class View implements ViewType {
 
     await NocoCache.del(deleteKeys);
   }
+
+  static async bulkColumnInsertToViews(
+    {
+      columns,
+      viewColumns,
+    }: {
+      columns?: ({
+        order?: number;
+        show?;
+      } & Column)[];
+      viewColumns?: (
+        | GridViewColumn
+        | GalleryViewColumn
+        | FormViewColumn
+        | KanbanViewColumn
+        | MapViewColumn
+      )[];
+    },
+    view: View,
+    ncMeta = Noco.ncMeta,
+  ) {
+    const insertObjs = [];
+
+    if (viewColumns) {
+      for (let i = 0; i < viewColumns.length; i++) {
+        const column = columns[i];
+
+        insertObjs.push({
+          ...column,
+          fk_view_id: view.id,
+          base_id: view.base_id,
+          source_id: view.source_id,
+        });
+      }
+    } else {
+      if (!columns) {
+        columns = await Column.list({ fk_model_id: view.fk_model_id });
+      }
+
+      for (let i = 0; i < columns.length; i++) {
+        const column = columns[i];
+
+        insertObjs.push({
+          fk_column_id: column.id,
+          order: column.order ?? i + 1,
+          show: column.show ?? true,
+          fk_view_id: view.id,
+          base_id: view.base_id,
+          source_id: view.source_id,
+        });
+      }
+    }
+
+    switch (view.type) {
+      case ViewTypes.GRID:
+        await ncMeta.bulkMetaInsert(
+          null,
+          null,
+          MetaTable.GRID_VIEW_COLUMNS,
+          insertObjs,
+        );
+        break;
+      case ViewTypes.GALLERY:
+        await ncMeta.bulkMetaInsert(
+          null,
+          null,
+          MetaTable.GALLERY_VIEW_COLUMNS,
+          insertObjs,
+        );
+        break;
+      case ViewTypes.MAP:
+        await ncMeta.bulkMetaInsert(
+          null,
+          null,
+          MetaTable.MAP_VIEW_COLUMNS,
+          insertObjs,
+        );
+        break;
+      case ViewTypes.KANBAN:
+        await ncMeta.bulkMetaInsert(
+          null,
+          null,
+          MetaTable.KANBAN_VIEW_COLUMNS,
+          insertObjs,
+        );
+        break;
+    }
+  }
+
+  static async insertMetaOnly(
+    view: Partial<View> &
+      Partial<FormView | GridView | GalleryView | KanbanView | MapView> & {
+        copy_from_id?: string;
+        fk_grp_col_id?: string;
+      },
+    model: {
+      getColumns: () => Promise<Column[]>;
+    },
+    ncMeta = Noco.ncMeta,
+  ) {
+    const insertObj = extractProps(view, [
+      'id',
+      'title',
+      'is_default',
+      'type',
+      'fk_model_id',
+      'base_id',
+      'source_id',
+      'meta',
+    ]);
+
+    if (!insertObj.order) {
+      // get order value
+      insertObj.order = await ncMeta.metaGetNextOrder(MetaTable.VIEWS, {
+        fk_model_id: view.fk_model_id,
+      });
+    }
+
+    insertObj.show = true;
+
+    if (!insertObj.meta) {
+      insertObj.meta = {};
+    }
+
+    insertObj.meta = stringifyMetaProp(insertObj);
+
+    const copyFromView =
+      view.copy_from_id && (await View.get(view.copy_from_id, ncMeta));
+
+    // get base and base id if missing
+    if (!(view.base_id && view.source_id)) {
+      const model = await Model.getByIdOrName({ id: view.fk_model_id }, ncMeta);
+      insertObj.base_id = model.base_id;
+      insertObj.source_id = model.source_id;
+    }
+
+    const insertedView = await ncMeta.metaInsert2(
+      null,
+      null,
+      MetaTable.VIEWS,
+      insertObj,
+    );
+
+    const { id: view_id } = insertedView;
+
+    // insert view metadata based on view type
+    switch (view.type) {
+      case ViewTypes.GRID:
+        await GridView.insert(
+          {
+            ...((copyFromView?.view as GridView) || {}),
+            ...(view as GridView),
+            fk_view_id: view_id,
+          },
+          ncMeta,
+        );
+        break;
+      case ViewTypes.MAP:
+        await MapView.insert(
+          {
+            ...(view as MapView),
+            fk_view_id: view_id,
+          },
+          ncMeta,
+        );
+        break;
+      case ViewTypes.GALLERY:
+        await GalleryView.insert(
+          {
+            ...(copyFromView?.view || {}),
+            ...view,
+            fk_view_id: view_id,
+          },
+          ncMeta,
+        );
+        break;
+      case ViewTypes.FORM:
+        await FormView.insert(
+          {
+            heading: view.title,
+            ...(copyFromView?.view || {}),
+            ...view,
+            fk_view_id: view_id,
+          },
+          ncMeta,
+        );
+        break;
+      case ViewTypes.KANBAN:
+        // set grouping field
+        (view as KanbanView).fk_grp_col_id = view.fk_grp_col_id;
+
+        await KanbanView.insert(
+          {
+            ...(copyFromView?.view || {}),
+            ...view,
+            fk_view_id: view_id,
+          },
+          ncMeta,
+        );
+        break;
+    }
+
+    //  copy from view
+    if (copyFromView) {
+      const sorts = await copyFromView.getSorts(ncMeta);
+      const filters = await Filter.rootFilterList(
+        { viewId: copyFromView.id },
+        ncMeta,
+      );
+      const viewColumns = await copyFromView.getColumns(ncMeta);
+
+      const sortInsertObjs = [];
+      const filterInsertObjs = [];
+
+      for (const sort of sorts) {
+        sortInsertObjs.push({
+          ...sort,
+          fk_view_id: view_id,
+          id: undefined,
+        });
+      }
+
+      for (const filter of filters) {
+        const fn = async (filter, parentId: string = null) => {
+          const generatedId = await ncMeta.genNanoid(MetaTable.FILTER_EXP);
+
+          const { children, ...filterProps } = filter;
+
+          filterInsertObjs.push({
+            ...filterProps,
+            fk_view_id: view_id,
+            id: generatedId,
+            fk_parent_id: parentId,
+          });
+          if (filter.is_group)
+            await Promise.all(
+              ((await filter.getChildren()) || []).map(async (child) => {
+                await fn(child, generatedId);
+              }),
+            );
+        };
+
+        await fn(filter);
+      }
+
+      await ncMeta.bulkMetaInsert(null, null, MetaTable.SORT, sortInsertObjs);
+
+      await ncMeta.bulkMetaInsert(
+        null,
+        null,
+        MetaTable.FILTER_EXP,
+        filterInsertObjs,
+        true,
+      );
+
+      // populate view columns
+      await View.bulkColumnInsertToViews({ viewColumns }, insertedView);
+    } else {
+      // populate view columns
+      await View.bulkColumnInsertToViews(
+        { columns: (await model.getColumns()) as any[] },
+        insertedView,
+      );
+    }
+
+    return insertedView;
+  }
 }

packages/nocodb/src/services/forms.service.ts

@@ -9,7 +9,9 @@ import type { NcRequest } from '~/interface/config';
 import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
 import { validatePayload } from '~/helpers';
 import { NcError } from '~/helpers/catchError';
-import { FormView, View } from '~/models';
+import { FormView, Model, View } from '~/models';
+import NocoCache from '~/cache/NocoCache';
+import { CacheScope } from '~/utils/globals';
 
 @Injectable()
 export class FormsService {
@@ -31,22 +33,39 @@ export class FormsService {
       param.body,
     );
 
-    const view = await View.insert({
+    // const view = await View.insert({
+    //   ...param.body,
+    //   // todo: sanitize
+    //   fk_model_id: param.tableId,
+    //   type: ViewTypes.FORM,
+    // });
+
+    const model = await Model.get(param.tableId);
+
+    const { id } = await View.insertMetaOnly(
+      {
         ...param.body,
         // todo: sanitize
         fk_model_id: param.tableId,
         type: ViewTypes.FORM,
-    });
+        base_id: model.base_id,
+        source_id: model.source_id,
+      },
+      model,
+    );
 
-    this.appHooksService.emit(AppEvents.VIEW_CREATE, {
-      view,
-      showAs: 'form',
-      req: param.req,
-    });
+    // populate  cache and add to list since the list cache already exist
+    const view = await View.get(id);
+    await NocoCache.appendToList(
+      CacheScope.VIEW,
+      [view.fk_model_id],
+      `${CacheScope.VIEW}:${id}`,
+    );
 
     this.appHooksService.emit(AppEvents.VIEW_CREATE, {
       user: param.user,
       view,
+      showAs: 'form',
       req: param.req,
     });
 

packages/nocodb/src/services/galleries.service.ts

@@ -9,7 +9,9 @@ import type { NcRequest } from '~/interface/config';
 import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
 import { validatePayload } from '~/helpers';
 import { NcError } from '~/helpers/catchError';
-import { GalleryView, View } from '~/models';
+import { GalleryView, Model, View } from '~/models';
+import NocoCache from '~/cache/NocoCache';
+import { CacheScope } from '~/utils/globals';
 
 @Injectable()
 export class GalleriesService {
@@ -30,13 +32,35 @@ export class GalleriesService {
       'swagger.json#/components/schemas/ViewCreateReq',
       param.gallery,
     );
+    //
+    // const view = await View.insert({
+    //   ...param.gallery,
+    //   // todo: sanitize
+    //   fk_model_id: param.tableId,
+    //   type: ViewTypes.GALLERY,
+    // });
 
-    const view = await View.insert({
+    const model = await Model.get(param.tableId);
+
+    const { id } = await View.insertMetaOnly(
+      {
         ...param.gallery,
         // todo: sanitize
         fk_model_id: param.tableId,
         type: ViewTypes.GALLERY,
-    });
+        base_id: model.base_id,
+        source_id: model.source_id,
+      },
+      model,
+    );
+
+    // populate  cache and add to list since the list cache already exist
+    const view = await View.get(id);
+    await NocoCache.appendToList(
+      CacheScope.VIEW,
+      [view.fk_model_id],
+      `${CacheScope.VIEW}:${id}`,
+    );
 
     this.appHooksService.emit(AppEvents.VIEW_CREATE, {
       view,

packages/nocodb/src/services/grids.service.ts

@@ -5,7 +5,9 @@ import type { NcRequest } from '~/interface/config';
 import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
 import { validatePayload } from '~/helpers';
 import { NcError } from '~/helpers/catchError';
-import { GridView, View } from '~/models';
+import { Column, GridView, Model, View } from '~/models';
+import NocoCache from '~/cache/NocoCache';
+import { CacheScope } from '~/utils/globals';
 
 @Injectable()
 export class GridsService {
@@ -20,13 +22,35 @@ export class GridsService {
       'swagger.json#/components/schemas/ViewCreateReq',
       param.grid,
     );
+    //
+    // const view = await View.insert({
+    //   ...param.grid,
+    //   // todo: sanitize
+    //   fk_model_id: param.tableId,
+    //   type: ViewTypes.GRID,
+    // });
 
-    const view = await View.insert({
+    const model = await Model.get(param.tableId);
+
+    const { id } = await View.insertMetaOnly(
+      {
         ...param.grid,
         // todo: sanitize
         fk_model_id: param.tableId,
         type: ViewTypes.GRID,
-    });
+        base_id: model.base_id,
+        source_id: model.source_id,
+      },
+      model,
+    );
+
+    // populate  cache and add to list since the list cache already exist
+    const view = await View.get(id);
+    await NocoCache.appendToList(
+      CacheScope.VIEW,
+      [view.fk_model_id],
+      `${CacheScope.VIEW}:${id}`,
+    );
 
     this.appHooksService.emit(AppEvents.VIEW_CREATE, {
       view,

packages/nocodb/src/services/kanbans.service.ts

@@ -9,7 +9,9 @@ import type { NcRequest } from '~/interface/config';
 import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
 import { validatePayload } from '~/helpers';
 import { NcError } from '~/helpers/catchError';
-import { KanbanView, View } from '~/models';
+import { KanbanView, Model, View } from '~/models'
+import NocoCache from '~/cache/NocoCache'
+import { CacheScope } from '~/utils/globals'
 
 @Injectable()
 export class KanbansService {
@@ -30,12 +32,34 @@ export class KanbansService {
       param.kanban,
     );
 
-    const view = await View.insert({
+    // const view = await View.insert({
+    //   ...param.kanban,
+    //   // todo: sanitize
+    //   fk_model_id: param.tableId,
+    //   type: ViewTypes.KANBAN,
+    // });
+
+    const model = await Model.get(param.tableId);
+
+    const { id } = await View.insertMetaOnly(
+      {
         ...param.kanban,
         // todo: sanitize
         fk_model_id: param.tableId,
         type: ViewTypes.KANBAN,
-    });
+        base_id: model.base_id,
+        source_id: model.source_id,
+      },
+      model,
+    );
+
+    // populate  cache and add to list since the list cache already exist
+    const view = await View.get(id);
+    await NocoCache.appendToList(
+      CacheScope.VIEW,
+      [view.fk_model_id],
+      `${CacheScope.VIEW}:${id}`,
+    );
 
     this.appHooksService.emit(AppEvents.VIEW_CREATE, {
       view,

packages/nocodb/src/services/maps.service.ts

@@ -5,7 +5,9 @@ import type { NcRequest } from '~/interface/config';
 import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
 import { validatePayload } from '~/helpers';
 import { NcError } from '~/helpers/catchError';
-import { MapView, View } from '~/models';
+import { MapView, Model, View } from '~/models';
+import { CacheScope } from '~/utils/globals';
+import NocoCache from '~/cache/NocoCache';
 
 @Injectable()
 export class MapsService {
@@ -25,12 +27,34 @@ export class MapsService {
       'swagger.json#/components/schemas/ViewCreateReq',
       param.map,
     );
-    const view = await View.insert({
+    // const view = await View.insert({
+    //   ...param.map,
+    //   // todo: sanitize
+    //   fk_model_id: param.tableId,
+    //   type: ViewTypes.MAP,
+    // });
+
+    const model = await Model.get(param.tableId);
+
+    const { id } = await View.insertMetaOnly(
+      {
         ...param.map,
         // todo: sanitize
         fk_model_id: param.tableId,
         type: ViewTypes.MAP,
-    });
+        base_id: model.base_id,
+        source_id: model.source_id,
+      },
+      model,
+    );
+
+    // populate  cache and add to list since the list cache already exist
+    const view = await View.get(id);
+    await NocoCache.appendToList(
+      CacheScope.VIEW,
+      [view.fk_model_id],
+      `${CacheScope.VIEW}:${id}`,
+    );
 
     this.appHooksService.emit(AppEvents.VIEW_CREATE, {
       view,