Browse Source

refactor: bulk meta insert

pull/7477/head
Pranav C 10 months ago
parent
commit
4321fa7f28
  1. 2
      packages/nocodb/src/cache/RedisCacheMgr.ts
  2. 19
      packages/nocodb/src/meta/meta.service.ts
  3. 226
      packages/nocodb/src/models/Column.ts
  4. 47
      packages/nocodb/src/models/Model.ts
  5. 267
      packages/nocodb/src/models/View.ts
  6. 43
      packages/nocodb/src/services/forms.service.ts
  7. 38
      packages/nocodb/src/services/galleries.service.ts
  8. 38
      packages/nocodb/src/services/grids.service.ts
  9. 38
      packages/nocodb/src/services/kanbans.service.ts
  10. 38
      packages/nocodb/src/services/maps.service.ts

2
packages/nocodb/src/cache/RedisCacheMgr.ts vendored

@ -23,7 +23,7 @@ export default class RedisCacheMgr extends CacheMgr {
process.env.NC_CLOUD !== 'true' process.env.NC_CLOUD !== 'true'
) { ) {
// flush the existing db with selected key (Default: 0) // flush the existing db with selected key (Default: 0)
this.client.flushdb(); // this.client.flushdb();
} }
// TODO(cache): fetch orgs once it's implemented // TODO(cache): fetch orgs once it's implemented

19
packages/nocodb/src/meta/meta.service.ts

@ -118,26 +118,31 @@ export class MetaService {
}); });
return insertObj; return insertObj;
} }
public async bulkMetaInsert( public async bulkMetaInsert(
base_id: string, base_id: string,
source_id: string, source_id: string,
target: string, target: string,
data: any[], data: any | any[],
ignoreIdGeneration?: boolean, ignoreIdGeneration?: boolean,
): Promise<any> { ): Promise<any> {
const insertObj = []; const insertObj = [];
const at = this.now(); const at = this.now();
for (const d of data) {
const commonProps: Record<string, any> = {
created_at: at,
updated_at: at,
};
if (source_id !== null) commonProps.source_id = source_id;
if (base_id !== null) commonProps.base_id = base_id;
for (const d of Array.isArray(data) ? data : [data]) {
const id = d?.id || (await this.genNanoid(target)); const id = d?.id || (await this.genNanoid(target));
const tempObj = { const tempObj = {
...d, ...d,
...(ignoreIdGeneration ? {} : { id }), ...(ignoreIdGeneration ? {} : { id }),
created_at: at, ...commonProps,
updated_at: at,
}; };
if (source_id !== null) tempObj.source_id = source_id;
if (base_id !== null) tempObj.base_id = base_id;
insertObj.push(tempObj); insertObj.push(tempObj);
} }

226
packages/nocodb/src/models/Column.ts

@ -1330,4 +1330,230 @@ export default class Column<T = any> implements ColumnType {
colId, colId,
); );
} }
static async bulkInsert(
param: {
columns: Column[];
fk_model_id: any;
source_id: string;
base_id: string;
},
ncMeta: MetaService = Noco.ncMeta,
) {
const extractedColumnMetas = [];
// add fk_model_id
for (const column of param.columns) {
extractedColumnMetas.push({
...extractProps(column as any, [
'id',
'fk_model_id',
'column_name',
'title',
'uidt',
'dt',
'np',
'ns',
'clen',
'cop',
'pk',
'rqd',
'un',
'ct',
'ai',
'unique',
'cdf',
'cc',
'csn',
'dtx',
'dtxp',
'dtxs',
'au',
'pv',
'order',
'base_id',
'source_id',
'system',
'meta',
]),
fk_model_id: param.fk_model_id,
});
}
// bulk insert columns
const columns = await ncMeta.bulkMetaInsert(
null,
null,
MetaTable.COLUMNS,
extractedColumnMetas,
);
// insert column options if any
// for (const column of columns) {
await Column.bulkInsertColOption(columns, ncMeta);
// }
return columns;
}
private static async bulkInsertColOption<T>(
columns: (Partial<T> & { source_id?: string; [p: string]: any })[],
ncMeta = Noco.ncMeta,
) {
const insertGroups = new Map<UITypes, Record<string, any>[]>();
for (const column of columns) {
let insertArr = insertGroups.get(column.uidt=== UITypes.MultiSelect ? UITypes.SingleSelect : column.uidt );
if (!insertArr) {
insertGroups.set(column.uidt, (insertArr = []));
}
switch (column.uidt || column.ui_data_type) {
case UITypes.Lookup:
// LookupColumn.insert()
insertArr.push({
fk_column_id: column.id,
fk_relation_column_id: column.fk_relation_column_id,
fk_lookup_column_id: column.fk_lookup_column_id,
});
break;
case UITypes.Rollup: {
insertArr.push({
fk_column_id: column.id,
fk_relation_column_id: column.fk_relation_column_id,
fk_rollup_column_id: column.fk_rollup_column_id,
rollup_function: column.rollup_function,
});
break;
}
case UITypes.Links:
case UITypes.LinkToAnotherRecord: {
insertArr.push({
fk_column_id: column.id,
// ref_db_alias
type: column.type,
// db_type:
fk_child_column_id: column.fk_child_column_id,
fk_parent_column_id: column.fk_parent_column_id,
fk_mm_model_id: column.fk_mm_model_id,
fk_mm_child_column_id: column.fk_mm_child_column_id,
fk_mm_parent_column_id: column.fk_mm_parent_column_id,
ur: column.ur,
dr: column.dr,
fk_index_name: column.fk_index_name,
fk_related_model_id: column.fk_related_model_id,
virtual: column.virtual,
});
break;
}
case UITypes.QrCode: {
insertArr.push(
{
fk_column_id: column.id,
fk_qr_value_column_id: column.fk_qr_value_column_id,
},
ncMeta,
);
break;
}
case UITypes.Barcode: {
insertArr.push({
fk_column_id: column.id,
fk_barcode_value_column_id: column.fk_barcode_value_column_id,
barcode_format: column.barcode_format,
});
break;
}
case UITypes.Formula: {
insertArr.push({
fk_column_id: column.id,
formula: column.formula,
formula_raw: column.formula_raw,
parsed_tree: column.parsed_tree,
});
break;
}
case UITypes.MultiSelect: {
if (!column.colOptions?.options) {
for (const [i, option] of column.dtxp?.split(',').entries() ||
[].entries()) {
insertArr.push({
fk_column_id: column.id,
title: option.replace(/^'/, '').replace(/'$/, ''),
order: i + 1,
color: selectColors[i % selectColors.length],
});
}
} else {
for (const [i, option] of column.colOptions.options.entries() ||
[].entries()) {
// Trim end of enum/set
if (column.dt === 'enum' || column.dt === 'set') {
option.title = option.title.trimEnd();
}
insertArr.push({
color: selectColors[i % selectColors.length], // in case color is not provided
...option,
fk_column_id: column.id,
order: i + 1,
});
}
}
break;
}
case UITypes.SingleSelect: {
if (!column.colOptions?.options) {
for (const [i, option] of column.dtxp?.split(',').entries() ||
[].entries()) {
insertArr.push({
fk_column_id: column.id,
title: option.replace(/^'/, '').replace(/'$/, ''),
order: i + 1,
color: selectColors[i % selectColors.length],
});
}
} else {
for (const [i, option] of column.colOptions.options.entries() ||
[].entries()) {
// Trim end of enum/set
if (column.dt === 'enum' || column.dt === 'set') {
option.title = option.title.trimEnd();
}
insertArr.push({
color: selectColors[i % selectColors.length], // in case color is not provided
...option,
fk_column_id: column.id,
order: i + 1,
});
}
}
break;
}
}
}
// bulk insert column options
for (const group of insertGroups.keys()) {
switch (group) {
case UITypes.SingleSelect:
case UITypes.MultiSelect:
await ncMeta.bulkMetaInsert(
null,
null,
MetaTable.COL_SELECT_OPTIONS,
insertGroups.get(group),
);
break;
// todo: handle rest of the cases
}
}
}
} }

47
packages/nocodb/src/models/Model.ts

@ -147,37 +147,50 @@ export default class Model implements TableType {
insertObj, insertObj,
); );
const view = await View.insert( const insertedColumns = await Column.bulkInsert(
{
columns: (model?.columns || []) as Column[],
fk_model_id: id,
source_id: sourceId,
base_id: baseId,
},
ncMeta,
);
await View.insertMetaOnly(
{ {
fk_model_id: id, fk_model_id: id,
title: model.title || model.table_name, title: model.title || model.table_name,
is_default: true, is_default: true,
type: ViewTypes.GRID, type: ViewTypes.GRID,
base_id: baseId,
source_id: sourceId,
},
{
getColumns: () => insertedColumns,
}, },
ncMeta, ncMeta,
); );
for (const column of model?.columns || []) { const model = await this.getWithInfo({ id }, ncMeta);
await Column.insert({ ...column, fk_model_id: id, view } as any, ncMeta);
}
return this.getWithInfo({ id }, ncMeta).then(async (model) => { // append to model list since model list cache will be there already
if (sourceId) { if (sourceId) {
await NocoCache.appendToList(
CacheScope.MODEL,
[baseId, sourceId],
`${CacheScope.MODEL}:${id}`,
);
}
// cater cases where sourceId is not required
// e.g. xcVisibilityMetaGet
await NocoCache.appendToList( await NocoCache.appendToList(
CacheScope.MODEL, CacheScope.MODEL,
[baseId], [baseId, sourceId],
`${CacheScope.MODEL}:${id}`, `${CacheScope.MODEL}:${id}`,
); );
return model; }
}); // cater cases where sourceId is not required
// e.g. xcVisibilityMetaGet
await NocoCache.appendToList(
CacheScope.MODEL,
[baseId],
`${CacheScope.MODEL}:${id}`,
);
return model;
} }
public static async list( public static async list(

267
packages/nocodb/src/models/View.ts

@ -1539,4 +1539,271 @@ export default class View implements ViewType {
await NocoCache.del(deleteKeys); await NocoCache.del(deleteKeys);
} }
static async bulkColumnInsertToViews(
{
columns,
viewColumns,
}: {
columns?: ({
order?: number;
show?;
} & Column)[];
viewColumns?: (
| GridViewColumn
| GalleryViewColumn
| FormViewColumn
| KanbanViewColumn
| MapViewColumn
)[];
},
view: View,
ncMeta = Noco.ncMeta,
) {
const insertObjs = [];
if (viewColumns) {
for (let i = 0; i < viewColumns.length; i++) {
const column = columns[i];
insertObjs.push({
...column,
fk_view_id: view.id,
base_id: view.base_id,
source_id: view.source_id,
});
}
} else {
if (!columns) {
columns = await Column.list({ fk_model_id: view.fk_model_id });
}
for (let i = 0; i < columns.length; i++) {
const column = columns[i];
insertObjs.push({
fk_column_id: column.id,
order: column.order ?? i + 1,
show: column.show ?? true,
fk_view_id: view.id,
base_id: view.base_id,
source_id: view.source_id,
});
}
}
switch (view.type) {
case ViewTypes.GRID:
await ncMeta.bulkMetaInsert(
null,
null,
MetaTable.GRID_VIEW_COLUMNS,
insertObjs,
);
break;
case ViewTypes.GALLERY:
await ncMeta.bulkMetaInsert(
null,
null,
MetaTable.GALLERY_VIEW_COLUMNS,
insertObjs,
);
break;
case ViewTypes.MAP:
await ncMeta.bulkMetaInsert(
null,
null,
MetaTable.MAP_VIEW_COLUMNS,
insertObjs,
);
break;
case ViewTypes.KANBAN:
await ncMeta.bulkMetaInsert(
null,
null,
MetaTable.KANBAN_VIEW_COLUMNS,
insertObjs,
);
break;
}
}
static async insertMetaOnly(
view: Partial<View> &
Partial<FormView | GridView | GalleryView | KanbanView | MapView> & {
copy_from_id?: string;
fk_grp_col_id?: string;
},
model: {
getColumns: () => Promise<Column[]>;
},
ncMeta = Noco.ncMeta,
) {
const insertObj = extractProps(view, [
'id',
'title',
'is_default',
'type',
'fk_model_id',
'base_id',
'source_id',
'meta',
]);
if (!insertObj.order) {
// get order value
insertObj.order = await ncMeta.metaGetNextOrder(MetaTable.VIEWS, {
fk_model_id: view.fk_model_id,
});
}
insertObj.show = true;
if (!insertObj.meta) {
insertObj.meta = {};
}
insertObj.meta = stringifyMetaProp(insertObj);
const copyFromView =
view.copy_from_id && (await View.get(view.copy_from_id, ncMeta));
// get base and base id if missing
if (!(view.base_id && view.source_id)) {
const model = await Model.getByIdOrName({ id: view.fk_model_id }, ncMeta);
insertObj.base_id = model.base_id;
insertObj.source_id = model.source_id;
}
const insertedView = await ncMeta.metaInsert2(
null,
null,
MetaTable.VIEWS,
insertObj,
);
const { id: view_id } = insertedView;
// insert view metadata based on view type
switch (view.type) {
case ViewTypes.GRID:
await GridView.insert(
{
...((copyFromView?.view as GridView) || {}),
...(view as GridView),
fk_view_id: view_id,
},
ncMeta,
);
break;
case ViewTypes.MAP:
await MapView.insert(
{
...(view as MapView),
fk_view_id: view_id,
},
ncMeta,
);
break;
case ViewTypes.GALLERY:
await GalleryView.insert(
{
...(copyFromView?.view || {}),
...view,
fk_view_id: view_id,
},
ncMeta,
);
break;
case ViewTypes.FORM:
await FormView.insert(
{
heading: view.title,
...(copyFromView?.view || {}),
...view,
fk_view_id: view_id,
},
ncMeta,
);
break;
case ViewTypes.KANBAN:
// set grouping field
(view as KanbanView).fk_grp_col_id = view.fk_grp_col_id;
await KanbanView.insert(
{
...(copyFromView?.view || {}),
...view,
fk_view_id: view_id,
},
ncMeta,
);
break;
}
// copy from view
if (copyFromView) {
const sorts = await copyFromView.getSorts(ncMeta);
const filters = await Filter.rootFilterList(
{ viewId: copyFromView.id },
ncMeta,
);
const viewColumns = await copyFromView.getColumns(ncMeta);
const sortInsertObjs = [];
const filterInsertObjs = [];
for (const sort of sorts) {
sortInsertObjs.push({
...sort,
fk_view_id: view_id,
id: undefined,
});
}
for (const filter of filters) {
const fn = async (filter, parentId: string = null) => {
const generatedId = await ncMeta.genNanoid(MetaTable.FILTER_EXP);
const { children, ...filterProps } = filter;
filterInsertObjs.push({
...filterProps,
fk_view_id: view_id,
id: generatedId,
fk_parent_id: parentId,
});
if (filter.is_group)
await Promise.all(
((await filter.getChildren()) || []).map(async (child) => {
await fn(child, generatedId);
}),
);
};
await fn(filter);
}
await ncMeta.bulkMetaInsert(null, null, MetaTable.SORT, sortInsertObjs);
await ncMeta.bulkMetaInsert(
null,
null,
MetaTable.FILTER_EXP,
filterInsertObjs,
true,
);
// populate view columns
await View.bulkColumnInsertToViews({ viewColumns }, insertedView);
} else {
// populate view columns
await View.bulkColumnInsertToViews(
{ columns: (await model.getColumns()) as any[] },
insertedView,
);
}
return insertedView;
}
} }

43
packages/nocodb/src/services/forms.service.ts

@ -9,7 +9,9 @@ import type { NcRequest } from '~/interface/config';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
import { validatePayload } from '~/helpers'; import { validatePayload } from '~/helpers';
import { NcError } from '~/helpers/catchError'; import { NcError } from '~/helpers/catchError';
import { FormView, View } from '~/models'; import { FormView, Model, View } from '~/models';
import NocoCache from '~/cache/NocoCache';
import { CacheScope } from '~/utils/globals';
@Injectable() @Injectable()
export class FormsService { export class FormsService {
@ -31,22 +33,39 @@ export class FormsService {
param.body, param.body,
); );
const view = await View.insert({ // const view = await View.insert({
...param.body, // ...param.body,
// todo: sanitize // // todo: sanitize
fk_model_id: param.tableId, // fk_model_id: param.tableId,
type: ViewTypes.FORM, // type: ViewTypes.FORM,
}); // });
this.appHooksService.emit(AppEvents.VIEW_CREATE, { const model = await Model.get(param.tableId);
view,
showAs: 'form', const { id } = await View.insertMetaOnly(
req: param.req, {
}); ...param.body,
// todo: sanitize
fk_model_id: param.tableId,
type: ViewTypes.FORM,
base_id: model.base_id,
source_id: model.source_id,
},
model,
);
// populate cache and add to list since the list cache already exist
const view = await View.get(id);
await NocoCache.appendToList(
CacheScope.VIEW,
[view.fk_model_id],
`${CacheScope.VIEW}:${id}`,
);
this.appHooksService.emit(AppEvents.VIEW_CREATE, { this.appHooksService.emit(AppEvents.VIEW_CREATE, {
user: param.user, user: param.user,
view, view,
showAs: 'form',
req: param.req, req: param.req,
}); });

38
packages/nocodb/src/services/galleries.service.ts

@ -9,7 +9,9 @@ import type { NcRequest } from '~/interface/config';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
import { validatePayload } from '~/helpers'; import { validatePayload } from '~/helpers';
import { NcError } from '~/helpers/catchError'; import { NcError } from '~/helpers/catchError';
import { GalleryView, View } from '~/models'; import { GalleryView, Model, View } from '~/models';
import NocoCache from '~/cache/NocoCache';
import { CacheScope } from '~/utils/globals';
@Injectable() @Injectable()
export class GalleriesService { export class GalleriesService {
@ -30,13 +32,35 @@ export class GalleriesService {
'swagger.json#/components/schemas/ViewCreateReq', 'swagger.json#/components/schemas/ViewCreateReq',
param.gallery, param.gallery,
); );
//
// const view = await View.insert({
// ...param.gallery,
// // todo: sanitize
// fk_model_id: param.tableId,
// type: ViewTypes.GALLERY,
// });
const view = await View.insert({ const model = await Model.get(param.tableId);
...param.gallery,
// todo: sanitize const { id } = await View.insertMetaOnly(
fk_model_id: param.tableId, {
type: ViewTypes.GALLERY, ...param.gallery,
}); // todo: sanitize
fk_model_id: param.tableId,
type: ViewTypes.GALLERY,
base_id: model.base_id,
source_id: model.source_id,
},
model,
);
// populate cache and add to list since the list cache already exist
const view = await View.get(id);
await NocoCache.appendToList(
CacheScope.VIEW,
[view.fk_model_id],
`${CacheScope.VIEW}:${id}`,
);
this.appHooksService.emit(AppEvents.VIEW_CREATE, { this.appHooksService.emit(AppEvents.VIEW_CREATE, {
view, view,

38
packages/nocodb/src/services/grids.service.ts

@ -5,7 +5,9 @@ import type { NcRequest } from '~/interface/config';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
import { validatePayload } from '~/helpers'; import { validatePayload } from '~/helpers';
import { NcError } from '~/helpers/catchError'; import { NcError } from '~/helpers/catchError';
import { GridView, View } from '~/models'; import { Column, GridView, Model, View } from '~/models';
import NocoCache from '~/cache/NocoCache';
import { CacheScope } from '~/utils/globals';
@Injectable() @Injectable()
export class GridsService { export class GridsService {
@ -20,13 +22,35 @@ export class GridsService {
'swagger.json#/components/schemas/ViewCreateReq', 'swagger.json#/components/schemas/ViewCreateReq',
param.grid, param.grid,
); );
//
// const view = await View.insert({
// ...param.grid,
// // todo: sanitize
// fk_model_id: param.tableId,
// type: ViewTypes.GRID,
// });
const view = await View.insert({ const model = await Model.get(param.tableId);
...param.grid,
// todo: sanitize const { id } = await View.insertMetaOnly(
fk_model_id: param.tableId, {
type: ViewTypes.GRID, ...param.grid,
}); // todo: sanitize
fk_model_id: param.tableId,
type: ViewTypes.GRID,
base_id: model.base_id,
source_id: model.source_id,
},
model,
);
// populate cache and add to list since the list cache already exist
const view = await View.get(id);
await NocoCache.appendToList(
CacheScope.VIEW,
[view.fk_model_id],
`${CacheScope.VIEW}:${id}`,
);
this.appHooksService.emit(AppEvents.VIEW_CREATE, { this.appHooksService.emit(AppEvents.VIEW_CREATE, {
view, view,

38
packages/nocodb/src/services/kanbans.service.ts

@ -9,7 +9,9 @@ import type { NcRequest } from '~/interface/config';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
import { validatePayload } from '~/helpers'; import { validatePayload } from '~/helpers';
import { NcError } from '~/helpers/catchError'; import { NcError } from '~/helpers/catchError';
import { KanbanView, View } from '~/models'; import { KanbanView, Model, View } from '~/models'
import NocoCache from '~/cache/NocoCache'
import { CacheScope } from '~/utils/globals'
@Injectable() @Injectable()
export class KanbansService { export class KanbansService {
@ -30,12 +32,34 @@ export class KanbansService {
param.kanban, param.kanban,
); );
const view = await View.insert({ // const view = await View.insert({
...param.kanban, // ...param.kanban,
// todo: sanitize // // todo: sanitize
fk_model_id: param.tableId, // fk_model_id: param.tableId,
type: ViewTypes.KANBAN, // type: ViewTypes.KANBAN,
}); // });
const model = await Model.get(param.tableId);
const { id } = await View.insertMetaOnly(
{
...param.kanban,
// todo: sanitize
fk_model_id: param.tableId,
type: ViewTypes.KANBAN,
base_id: model.base_id,
source_id: model.source_id,
},
model,
);
// populate cache and add to list since the list cache already exist
const view = await View.get(id);
await NocoCache.appendToList(
CacheScope.VIEW,
[view.fk_model_id],
`${CacheScope.VIEW}:${id}`,
);
this.appHooksService.emit(AppEvents.VIEW_CREATE, { this.appHooksService.emit(AppEvents.VIEW_CREATE, {
view, view,

38
packages/nocodb/src/services/maps.service.ts

@ -5,7 +5,9 @@ import type { NcRequest } from '~/interface/config';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
import { validatePayload } from '~/helpers'; import { validatePayload } from '~/helpers';
import { NcError } from '~/helpers/catchError'; import { NcError } from '~/helpers/catchError';
import { MapView, View } from '~/models'; import { MapView, Model, View } from '~/models';
import { CacheScope } from '~/utils/globals';
import NocoCache from '~/cache/NocoCache';
@Injectable() @Injectable()
export class MapsService { export class MapsService {
@ -25,12 +27,34 @@ export class MapsService {
'swagger.json#/components/schemas/ViewCreateReq', 'swagger.json#/components/schemas/ViewCreateReq',
param.map, param.map,
); );
const view = await View.insert({ // const view = await View.insert({
...param.map, // ...param.map,
// todo: sanitize // // todo: sanitize
fk_model_id: param.tableId, // fk_model_id: param.tableId,
type: ViewTypes.MAP, // type: ViewTypes.MAP,
}); // });
const model = await Model.get(param.tableId);
const { id } = await View.insertMetaOnly(
{
...param.map,
// todo: sanitize
fk_model_id: param.tableId,
type: ViewTypes.MAP,
base_id: model.base_id,
source_id: model.source_id,
},
model,
);
// populate cache and add to list since the list cache already exist
const view = await View.get(id);
await NocoCache.appendToList(
CacheScope.VIEW,
[view.fk_model_id],
`${CacheScope.VIEW}:${id}`,
);
this.appHooksService.emit(AppEvents.VIEW_CREATE, { this.appHooksService.emit(AppEvents.VIEW_CREATE, {
view, view,

Loading…
Cancel
Save