diff --git a/packages/nocodb/src/controllers/exportImport/export.ctl.ts b/packages/nocodb/src/controllers/exportImport/export.ctl.ts new file mode 100644 index 0000000000..09fefad858 --- /dev/null +++ b/packages/nocodb/src/controllers/exportImport/export.ctl.ts @@ -0,0 +1,19 @@ +import { Router } from 'express'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import { exportService } from '../../services'; +import type { Request, Response } from 'express'; + +export async function exportModel(req: Request, res: Response) { + res.json( + await exportService.exportModel({ modelId: req.params.modelId.split(',') }) + ); +} + +const router = Router({ mergeParams: true }); + +router.get( + '/api/v1/db/meta/export/:modelId', + ncMetaAclMw(exportModel, 'exportModel') +); + +export default router; diff --git a/packages/nocodb/src/controllers/exportImport/import.ctl.ts b/packages/nocodb/src/controllers/exportImport/import.ctl.ts new file mode 100644 index 0000000000..80af204134 --- /dev/null +++ b/packages/nocodb/src/controllers/exportImport/import.ctl.ts @@ -0,0 +1,26 @@ +import { Router } from 'express'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import { importService } from '../../services'; +import type { Request, Response } from 'express'; + +export async function importModels(req: Request, res: Response) { + const { body, ...rest } = req; + res.json( + await importService.importModels({ + user: (req as any).user, + projectId: req.params.projectId, + baseId: req.params.baseId, + data: Array.isArray(body) ? body : body.models, + req: rest, + }) + ); +} + +const router = Router({ mergeParams: true }); + +router.post( + '/api/v1/db/meta/import/:projectId/:baseId', + ncMetaAclMw(importModels, 'importModels') +); + +export default router; diff --git a/packages/nocodb/src/controllers/exportImport/index.ts b/packages/nocodb/src/controllers/exportImport/index.ts new file mode 100644 index 0000000000..f2795eff50 --- /dev/null +++ b/packages/nocodb/src/controllers/exportImport/index.ts @@ -0,0 +1,7 @@ +import exportController from './export.ctl'; +import importController from './import.ctl'; + +export default { + exportController, + importController, +}; diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index 7e8f413898..4ae2e48ef0 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -50,6 +50,7 @@ import { import swaggerController from '../../controllers/apiDocs'; import { importController, syncSourceController } from '../../controllers/sync'; import mapViewController from '../../controllers/views/mapView.ctl'; +import exportImportController from '../../controllers/exportImport' import type { Socket } from 'socket.io'; import type { Router } from 'express'; @@ -103,6 +104,8 @@ export default function (router: Router, server) { router.use(syncSourceController); router.use(kanbanViewController); router.use(mapViewController); + router.use(exportImportController.exportController); + router.use(exportImportController.importController); userController(router); diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index d181c4587a..4c47a282cc 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -36,3 +36,5 @@ export * as syncService from './sync'; export * from './public'; export * as orgTokenService from './orgToken.svc'; export * as orgTokenServiceEE from './ee/orgToken.svc'; +export * as exportService from './exportImport/export.svc'; +export * as importService from './exportImport/import.svc'; diff --git a/packages/nocodb/src/services/exportImport/export.svc.ts b/packages/nocodb/src/services/exportImport/export.svc.ts new file mode 100644 index 0000000000..4c731c41ac --- /dev/null +++ b/packages/nocodb/src/services/exportImport/export.svc.ts @@ -0,0 +1,233 @@ +import { NcError } from './../../meta/helpers/catchError'; +import { ViewTypes } from 'nocodb-sdk'; +import { Project, Base, Model } from '../../models'; + +export async function exportModel(param: { modelId: string[] }) { + const exportData = { + models: [], + }; + + // db id to human readable id + const idMap = new Map(); + + const projects: Project[] = [] + const bases: Base[] = [] + const modelsMap = new Map(); + + for (const modelId of param.modelId) { + const model = await Model.get(modelId); + + if (!model) return NcError.badRequest(`Model not found for id '${modelId}'`); + + const fndProject = projects.find(p => p.id === model.project_id) + const project = fndProject || await Project.get(model.project_id); + + const fndBase = bases.find(b => b.id === model.base_id) + const base = fndBase || await Base.get(model.base_id); + + if (!fndProject) projects.push(project); + if (!fndBase) bases.push(base); + + if (!modelsMap.has(base.id)) { + const all_models = await base.getModels(); + + for (const md of all_models) { + idMap.set(md.id, `${project.title}::${base.alias || 'default'}::${clearPrefix(md.table_name, project.prefix)}`); + await md.getColumns(); + for (const column of md.columns) { + idMap.set(column.id, `${idMap.get(md.id)}::${column.column_name || column.title}`); + } + } + + modelsMap.set(base.id, all_models); + } + + idMap.set(project.id, project.title); + idMap.set(base.id, `${project.title}::${base.alias || 'default'}`); + idMap.set(model.id, `${idMap.get(base.id)}::${clearPrefix(model.table_name, project.prefix)}`); + + await model.getColumns(); + await model.getViews(); + + for (const column of model.columns) { + idMap.set( + column.id, + `${idMap.get(model.id)}::${column.column_name || column.title}` + ); + await column.getColOptions(); + if (column.colOptions) { + for (const [k, v] of Object.entries(column.colOptions)) { + switch (k) { + case 'fk_mm_child_column_id': + case 'fk_mm_parent_column_id': + case 'fk_mm_model_id': + case 'fk_parent_column_id': + case 'fk_child_column_id': + case 'fk_related_model_id': + case 'fk_relation_column_id': + case 'fk_lookup_column_id': + case 'fk_rollup_column_id': + column.colOptions[k] = idMap.get(v as string); + break; + case 'options': + for (const o of column.colOptions['options']) { + delete o.id; + delete o.fk_column_id; + } + break; + case 'formula': + column.colOptions[k] = column.colOptions[k].replace(/(?<=\{\{).*?(?=\}\})/gm, (match) => idMap.get(match)); + break; + case 'id': + case 'created_at': + case 'updated_at': + case 'fk_column_id': + delete column.colOptions[k]; + break; + } + } + } + } + + for (const view of model.views) { + idMap.set(view.id, `${idMap.get(model.id)}::${view.title}`); + await view.getColumns(); + await view.getFilters(); + await view.getSorts(); + if (view.filter) { + const export_filters = [] + for (const fl of view.filter.children) { + const tempFl = { + id: fl.id, + fk_column_id: idMap.get(fl.fk_column_id), + fk_parent_id: fl.fk_parent_id, + is_group: fl.is_group, + logical_op: fl.logical_op, + comparison_op: fl.comparison_op, + comparison_sub_op: fl.comparison_sub_op, + value: fl.value, + } + if (tempFl.is_group) { + delete tempFl.comparison_op; + delete tempFl.comparison_sub_op; + delete tempFl.value; + } + export_filters.push(tempFl) + } + view.filter.children = export_filters; + } + + if (view.sorts) { + const export_sorts = [] + for (const sr of view.sorts) { + const tempSr = { + fk_column_id: idMap.get(sr.fk_column_id), + direction: sr.direction, + } + export_sorts.push(tempSr) + } + view.sorts = export_sorts; + } + + if (view.view) { + for (const [k, v] of Object.entries(view.view)) { + switch (k) { + case 'fk_column_id': + case 'fk_cover_image_col_id': + case 'fk_grp_col_id': + view.view[k] = idMap.get(v as string); + break; + case 'meta': + if (view.type === ViewTypes.KANBAN) { + const meta = JSON.parse(view.view.meta as string) as Record; + for (const [k, v] of Object.entries(meta)) { + const colId = idMap.get(k as string); + for (const op of v) { + op.fk_column_id = idMap.get(op.fk_column_id); + delete op.id; + } + meta[colId] = v; + delete meta[k]; + } + view.view.meta = meta; + } + break; + case 'created_at': + case 'updated_at': + case 'fk_view_id': + case 'project_id': + case 'base_id': + case 'uuid': + delete view.view[k]; + break; + } + } + } + } + + exportData.models.push({ + model: { + id: idMap.get(model.id), + prefix: project.prefix, + title: model.title, + table_name: clearPrefix(model.table_name, project.prefix), + meta: model.meta, + columns: model.columns.map((column) => ({ + id: idMap.get(column.id), + ai: column.ai, + column_name: column.column_name, + cc: column.cc, + cdf: column.cdf, + meta: column.meta, + pk: column.pk, + order: column.order, + rqd: column.rqd, + system: column.system, + uidt: column.uidt, + title: column.title, + un: column.un, + unique: column.unique, + colOptions: column.colOptions, + })), + }, + views: model.views.map((view) => ({ + id: idMap.get(view.id), + is_default: view.is_default, + type: view.type, + meta: view.meta, + order: view.order, + title: view.title, + show: view.show, + show_system_fields: view.show_system_fields, + filter: view.filter, + sorts: view.sorts, + lock_type: view.lock_type, + columns: view.columns.map((column) => { + const { + id, + fk_view_id, + fk_column_id, + project_id, + base_id, + created_at, + updated_at, + uuid, + ...rest + } = column as any; + return { + fk_column_id: idMap.get(fk_column_id), + ...rest, + }; + }), + view: view.view, + })), + }); + } + + return exportData; +} + +const clearPrefix = (text: string, prefix?: string) => { + if (!prefix) return text; + return text.replace(new RegExp(`^${prefix}_`), ''); +} diff --git a/packages/nocodb/src/services/exportImport/import.svc.ts b/packages/nocodb/src/services/exportImport/import.svc.ts new file mode 100644 index 0000000000..cc72db8dcb --- /dev/null +++ b/packages/nocodb/src/services/exportImport/import.svc.ts @@ -0,0 +1,529 @@ +import type { ViewCreateReqType } from 'nocodb-sdk'; +import { UITypes, ViewTypes } from 'nocodb-sdk'; +import { tableService, gridViewService, filterService, viewColumnService, gridViewColumnService, sortService, formViewService, galleryViewService, kanbanViewService, formViewColumnService, columnService } from '..'; +import { NcError } from '../../meta/helpers/catchError'; +import { Project, Base, User, View, Model } from '../../models'; + +export async function importModels(param: { + user: User; + projectId: string; + baseId: string; + data: { model: any; views: any[] }[]; + req: any; +}) { + + // human readable id to db id + const idMap = new Map(); + + const project = await Project.get(param.projectId); + + if (!project) return NcError.badRequest(`Project not found for id '${param.projectId}'`); + + const base = await Base.get(param.baseId); + + if (!base) return NcError.badRequest(`Base not found for id '${param.baseId}'`); + + const tableReferences = new Map(); + const linkMap = new Map(); + + // create tables with static columns + for (const data of param.data) { + const modelData = data.model; + + const reducedColumnSet = modelData.columns.filter( + (a) => + a.uidt !== UITypes.LinkToAnotherRecord && + a.uidt !== UITypes.Lookup && + a.uidt !== UITypes.Rollup && + a.uidt !== UITypes.Formula && + a.uidt !== UITypes.ForeignKey + ); + + // create table with static columns + const table = await tableService.tableCreate({ + projectId: project.id, + baseId: base.id, + user: param.user, + table: withoutId({ + ...modelData, + columns: reducedColumnSet.map((a) => withoutId(a)), + }), + }); + + idMap.set(modelData.id, table.id); + + // map column id's with new created column id's + for (const col of table.columns) { + const colRef = modelData.columns.find( + (a) => a.column_name === col.column_name + ); + idMap.set(colRef.id, col.id); + } + + tableReferences.set(modelData.id, table); + } + + const referencedColumnSet = [] + + // create columns with reference to other columns + for (const data of param.data) { + const modelData = data.model; + const table = tableReferences.get(modelData.id); + + const linkedColumnSet = modelData.columns.filter( + (a) => a.uidt === UITypes.LinkToAnotherRecord + ); + + // create columns with reference to other columns + for (const col of linkedColumnSet) { + if (col.colOptions) { + const colOptions = col.colOptions; + if (col.uidt === UITypes.LinkToAnotherRecord && idMap.has(colOptions.fk_related_model_id)) { + if (colOptions.type === 'mm') { + if (!linkMap.has(colOptions.fk_mm_model_id)) { + // delete col.column_name as it is not required and will cause ajv error (null for LTAR) + delete col.column_name; + + const freshModelData = await columnService.columnAdd({ + tableId: table.id, + column: withoutId({ + ...col, + ...{ + parentId: idMap.get(getParentIdentifier(colOptions.fk_child_column_id)), + childId: idMap.get(getParentIdentifier(colOptions.fk_parent_column_id)), + type: colOptions.type, + virtual: colOptions.virtual, + ur: colOptions.ur, + dr: colOptions.dr, + }, + }), + req: param.req, + }); + + for (const nColumn of freshModelData.columns) { + if (nColumn.title === col.title) { + idMap.set(col.id, nColumn.id); + linkMap.set(colOptions.fk_mm_model_id, nColumn.colOptions.fk_mm_model_id); + break; + } + } + + const childModel = getParentIdentifier(colOptions.fk_parent_column_id) === modelData.id ? freshModelData : await Model.get(idMap.get(getParentIdentifier(colOptions.fk_parent_column_id))); + + if (getParentIdentifier(colOptions.fk_parent_column_id) !== modelData.id) await childModel.getColumns(); + + const childColumn = param.data.find(a => a.model.id === getParentIdentifier(colOptions.fk_parent_column_id)).model.columns.find(a => a.colOptions?.fk_mm_model_id === colOptions.fk_mm_model_id && a.id !== col.id); + + for (const nColumn of childModel.columns) { + if (nColumn?.colOptions?.fk_mm_model_id === linkMap.get(colOptions.fk_mm_model_id) && nColumn.id !== idMap.get(col.id)) { + idMap.set(childColumn.id, nColumn.id); + + await columnService.columnUpdate({ + columnId: nColumn.id, + column: { + ...nColumn, + column_name: childColumn.title, + title: childColumn.title, + }, + }); + break; + } + } + } + } else if (colOptions.type === 'hm') { + // delete col.column_name as it is not required and will cause ajv error (null for LTAR) + delete col.column_name; + + const freshModelData = await columnService.columnAdd({ + tableId: table.id, + column: withoutId({ + ...col, + ...{ + parentId: idMap.get(getParentIdentifier(colOptions.fk_parent_column_id)), + childId: idMap.get(getParentIdentifier(colOptions.fk_child_column_id)), + type: colOptions.type, + virtual: colOptions.virtual, + ur: colOptions.ur, + dr: colOptions.dr, + }, + }), + req: param.req, + }); + + for (const nColumn of freshModelData.columns) { + if (nColumn.title === col.title) { + idMap.set(col.id, nColumn.id); + linkMap.set(colOptions.fk_index_name, nColumn.colOptions.fk_index_name); + break; + } + } + + const childModel = colOptions.fk_related_model_id === modelData.id ? freshModelData : await Model.get(idMap.get(colOptions.fk_related_model_id)); + + if (colOptions.fk_related_model_id !== modelData.id) await childModel.getColumns(); + + const childColumn = param.data.find(a => a.model.id === colOptions.fk_related_model_id).model.columns.find(a => a.colOptions?.fk_index_name === colOptions.fk_index_name && a.id !== col.id); + + for (const nColumn of childModel.columns) { + if (nColumn?.colOptions?.fk_index_name === linkMap.get(colOptions.fk_index_name) && nColumn.id !== idMap.get(col.id)) { + idMap.set(childColumn.id, nColumn.id); + + await columnService.columnUpdate({ + columnId: nColumn.id, + column: { + ...nColumn, + column_name: childColumn.title, + title: childColumn.title, + }, + }); + break; + } + } + } + } + } + } + + referencedColumnSet.push(...modelData.columns.filter( + (a) => + a.uidt === UITypes.Lookup || + a.uidt === UITypes.Rollup || + a.uidt === UITypes.Formula + )); + } + + const sortedReferencedColumnSet = []; + + // sort referenced columns to avoid referencing before creation + for (const col of referencedColumnSet) { + const relatedColIds = []; + if (col.colOptions?.fk_lookup_column_id) { + relatedColIds.push(col.colOptions.fk_lookup_column_id); + } + if (col.colOptions?.fk_rollup_column_id) { + relatedColIds.push(col.colOptions.fk_rollup_column_id); + } + if (col.colOptions?.formula) { + relatedColIds.push(...col.colOptions.formula.match(/(?<=\{\{).*?(?=\}\})/gm)); + } + + // find the last related column in the sorted array + let fnd = undefined; + for (let i = sortedReferencedColumnSet.length - 1; i >= 0; i--) { + if (relatedColIds.includes(sortedReferencedColumnSet[i].id)) { + fnd = sortedReferencedColumnSet[i]; + break; + } + } + + if (!fnd) { + sortedReferencedColumnSet.unshift(col); + } else { + sortedReferencedColumnSet.splice(sortedReferencedColumnSet.indexOf(fnd) + 1, 0, col); + } + } + + // create referenced columns + for (const col of sortedReferencedColumnSet) { + const { colOptions, ...flatCol } = col; + if (col.uidt === UITypes.Lookup) { + const freshModelData = await columnService.columnAdd({ + tableId: idMap.get(getParentIdentifier(col.id)), + column: withoutId({ + ...flatCol, + ...{ + fk_lookup_column_id: idMap.get(colOptions.fk_lookup_column_id), + fk_relation_column_id: idMap.get(colOptions.fk_relation_column_id), + }, + }), + req: param.req, + }); + + for (const nColumn of freshModelData.columns) { + if (nColumn.title === col.title) { + idMap.set(col.id, nColumn.id); + break; + } + } + } else if (col.uidt === UITypes.Rollup) { + const freshModelData = await columnService.columnAdd({ + tableId: idMap.get(getParentIdentifier(col.id)), + column: withoutId({ + ...flatCol, + ...{ + fk_rollup_column_id: idMap.get(colOptions.fk_rollup_column_id), + fk_relation_column_id: idMap.get(colOptions.fk_relation_column_id), + rollup_function: colOptions.rollup_function, + }, + }), + req: param.req, + }); + + for (const nColumn of freshModelData.columns) { + if (nColumn.title === col.title) { + idMap.set(col.id, nColumn.id); + break; + } + } + } else if (col.uidt === UITypes.Formula) { + const freshModelData = await columnService.columnAdd({ + tableId: idMap.get(getParentIdentifier(col.id)), + column: withoutId({ + ...flatCol, + ...{ + formula_raw: colOptions.formula_raw, + }, + }), + req: param.req, + }); + + for (const nColumn of freshModelData.columns) { + if (nColumn.title === col.title) { + idMap.set(col.id, nColumn.id); + break; + } + } + } + } + + // create views + for (const data of param.data) { + const modelData = data.model; + const viewsData = data.views; + + const table = tableReferences.get(modelData.id); + + // get default view + await table.getViews(); + + for (const view of viewsData) { + const viewData = withoutId({ + ...view, + }); + + const vw = await createView(idMap, table, viewData, table.views); + + if (!vw) continue; + + idMap.set(view.id, vw.id); + + // create filters + const filters = view.filter.children; + + for (const fl of filters) { + const fg = await filterService.filterCreate({ + viewId: vw.id, + filter: withoutId({ + ...fl, + fk_column_id: idMap.get(fl.fk_column_id), + fk_parent_id: idMap.get(fl.fk_parent_id), + }), + }); + + idMap.set(fl.id, fg.id); + } + + // create sorts + for (const sr of view.sorts) { + await sortService.sortCreate({ + viewId: vw.id, + sort: withoutId({ + ...sr, + fk_column_id: idMap.get(sr.fk_column_id), + }), + }) + } + + // update view columns + const vwColumns = await viewColumnService.columnList({ viewId: vw.id }) + + for (const cl of vwColumns) { + const fcl = view.columns.find(a => a.fk_column_id === reverseGet(idMap, cl.fk_column_id)) + if (!fcl) continue; + await viewColumnService.columnUpdate({ + viewId: vw.id, + columnId: cl.id, + column: { + show: fcl.show, + order: fcl.order, + }, + }) + } + + switch (vw.type) { + case ViewTypes.GRID: + for (const cl of vwColumns) { + const fcl = view.columns.find(a => a.fk_column_id === reverseGet(idMap, cl.fk_column_id)) + if (!fcl) continue; + const { fk_column_id, ...rest } = fcl; + await gridViewColumnService.gridColumnUpdate({ + gridViewColumnId: cl.id, + grid: { + ...withoutNull(rest), + }, + }) + } + break; + case ViewTypes.FORM: + for (const cl of vwColumns) { + const fcl = view.columns.find(a => a.fk_column_id === reverseGet(idMap, cl.fk_column_id)) + if (!fcl) continue; + const { fk_column_id, ...rest } = fcl; + await formViewColumnService.columnUpdate({ + formViewColumnId: cl.id, + formViewColumn: { + ...withoutNull(rest), + }, + }) + } + break; + case ViewTypes.GALLERY: + case ViewTypes.KANBAN: + break; + } + } + } +} + +async function createView(idMap: Map, md: Model, vw: Partial, views: View[]): Promise { + if (vw.is_default) { + const view = views.find((a) => a.is_default); + if (view) { + const gridData = withoutNull(vw.view); + if (gridData) { + await gridViewService.gridViewUpdate({ + viewId: view.id, + grid: gridData, + }); + } + } + return view; + } + + switch (vw.type) { + case ViewTypes.GRID: + const gview = await gridViewService.gridViewCreate({ + tableId: md.id, + grid: vw as ViewCreateReqType, + }); + const gridData = withoutNull(vw.view); + if (gridData) { + await gridViewService.gridViewUpdate({ + viewId: gview.id, + grid: gridData, + }); + } + return gview; + case ViewTypes.FORM: + const fview = await formViewService.formViewCreate({ + tableId: md.id, + body: vw as ViewCreateReqType, + }); + const formData = withoutNull(vw.view); + if (formData) { + await formViewService.formViewUpdate({ + formViewId: fview.id, + form: formData, + }); + } + return fview; + case ViewTypes.GALLERY: + const glview = await galleryViewService.galleryViewCreate({ + tableId: md.id, + gallery: vw as ViewCreateReqType, + }); + const galleryData = withoutNull(vw.view); + if (galleryData) { + for (const [k, v] of Object.entries(galleryData)) { + switch (k) { + case 'fk_cover_image_col_id': + galleryData[k] = idMap.get(v as string); + break; + } + } + await galleryViewService.galleryViewUpdate({ + galleryViewId: glview.id, + gallery: galleryData, + }); + } + return glview; + case ViewTypes.KANBAN: + const kview = await kanbanViewService.kanbanViewCreate({ + tableId: md.id, + kanban: vw as ViewCreateReqType, + }); + const kanbanData = withoutNull(vw.view); + if (kanbanData) { + for (const [k, v] of Object.entries(kanbanData)) { + switch (k) { + case 'fk_grp_col_id': + case 'fk_cover_image_col_id': + kanbanData[k] = idMap.get(v as string); + break; + case 'meta': + const meta = {}; + for (const [mk, mv] of Object.entries(v as any)) { + const tempVal = []; + for (const vl of mv as any) { + if (vl.fk_column_id) { + tempVal.push({ + ...vl, + fk_column_id: idMap.get(vl.fk_column_id), + }); + } else { + delete vl.fk_column_id; + tempVal.push({ + ...vl, + id: "uncategorized", + }); + } + } + meta[idMap.get(mk)] = tempVal; + } + kanbanData[k] = meta; + break; + } + } + await kanbanViewService.kanbanViewUpdate({ + kanbanViewId: kview.id, + kanban: kanbanData, + }); + } + return kview; + } + + return null +} + +function withoutNull(obj: any) { + const newObj = {}; + let found = false; + for (const [key, value] of Object.entries(obj)) { + if (value !== null) { + newObj[key] = value; + found = true; + } + } + if (!found) return null; + return newObj; +} + +function reverseGet(map: Map, vl: string) { + for (const [key, value] of map.entries()) { + if (vl === value) { + return key; + } + } + return undefined +} + +function withoutId(obj: any) { + const { id, ...rest } = obj; + return rest; +} + +function getParentIdentifier(id: string) { + const arr = id.split('::'); + arr.pop(); + return arr.join('::'); +}