From 5912590dd747fd16c46637b241329b604519684b Mon Sep 17 00:00:00 2001 From: mertmit Date: Thu, 20 Apr 2023 05:55:59 +0300 Subject: [PATCH] feat: duplicate job Signed-off-by: mertmit --- packages/nocodb-nest/src/app.module.ts | 9 + .../export-import.controller.spec.ts | 21 - .../controllers/export-import.controller.ts | 51 -- .../src/helpers/exportImportHelpers.ts | 10 + .../export-import/duplicate.controller.ts | 42 ++ .../jobs/export-import/duplicate.processor.ts | 102 ++++ .../jobs/export-import/export.service.ts | 487 ++++++++++++++++ .../jobs/export-import/import.service.ts} | 548 +----------------- .../src/modules/jobs/jobs.module.ts | 23 + .../src/modules/metas/metas.module.ts | 20 +- .../services/export-import.service.spec.ts | 19 - 11 files changed, 714 insertions(+), 618 deletions(-) delete mode 100644 packages/nocodb-nest/src/controllers/export-import.controller.spec.ts delete mode 100644 packages/nocodb-nest/src/controllers/export-import.controller.ts create mode 100644 packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts create mode 100644 packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts create mode 100644 packages/nocodb-nest/src/modules/jobs/export-import/export.service.ts rename packages/nocodb-nest/src/{services/export-import.service.ts => modules/jobs/export-import/import.service.ts} (65%) create mode 100644 packages/nocodb-nest/src/modules/jobs/jobs.module.ts delete mode 100644 packages/nocodb-nest/src/services/export-import.service.spec.ts diff --git a/packages/nocodb-nest/src/app.module.ts b/packages/nocodb-nest/src/app.module.ts index 1307931d5d..1e479dc475 100644 --- a/packages/nocodb-nest/src/app.module.ts +++ b/packages/nocodb-nest/src/app.module.ts @@ -1,5 +1,6 @@ import { Module, RequestMethod } from '@nestjs/common'; import { APP_FILTER } from '@nestjs/core'; +import { BullModule } from '@nestjs/bull'; import { Connection } from './connection/connection'; import { GlobalExceptionFilter } from './filters/global-exception/global-exception.filter'; import NcPluginMgrv2 from './helpers/NcPluginMgrv2'; @@ -20,6 +21,7 @@ import NcConfigFactory from './utils/NcConfigFactory' import NcUpgrader from './version-upgrader/NcUpgrader'; import { MetasModule } from './modules/metas/metas.module'; import NocoCache from './cache/NocoCache'; +import { JobsModule } from './modules/jobs/jobs.module'; import type { MiddlewareConsumer, OnApplicationBootstrap, @@ -32,6 +34,13 @@ import type { ...(process.env['PLAYWRIGHT_TEST'] === 'true' ? [TestModule] : []), MetasModule, DatasModule, + JobsModule, + BullModule.forRoot({ + redis: { + host: 'localhost', + port: 6379, + }, + }), ], controllers: [], providers: [ diff --git a/packages/nocodb-nest/src/controllers/export-import.controller.spec.ts b/packages/nocodb-nest/src/controllers/export-import.controller.spec.ts deleted file mode 100644 index 02e75800f9..0000000000 --- a/packages/nocodb-nest/src/controllers/export-import.controller.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { ExportImportService } from './../services/export-import.service'; -import { ExportImportController } from './export-import.controller'; -import type { TestingModule } from '@nestjs/testing'; - -describe('ExportImportController', () => { - let controller: ExportImportController; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [ExportImportController], - providers: [ExportImportService], - }).compile(); - - controller = module.get(ExportImportController); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); -}); diff --git a/packages/nocodb-nest/src/controllers/export-import.controller.ts b/packages/nocodb-nest/src/controllers/export-import.controller.ts deleted file mode 100644 index 696b6e8a41..0000000000 --- a/packages/nocodb-nest/src/controllers/export-import.controller.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { - Body, - Controller, - HttpCode, - Param, - Patch, - Post, - Request, - UseGuards, -} from '@nestjs/common'; -import { AuthGuard } from '@nestjs/passport'; -import { GlobalGuard } from '../guards/global/global.guard'; -import { - Acl, - ExtractProjectIdMiddleware, -} from '../middlewares/extract-project-id/extract-project-id.middleware'; -import { ExportImportService } from './../services/export-import.service'; - -@Controller() -@UseGuards(ExtractProjectIdMiddleware, GlobalGuard) -export class ExportImportController { - constructor(private readonly exportImportService: ExportImportService) {} - - @Post('/api/v1/db/meta/export/:projectId/:baseId') - @HttpCode(200) - @Acl('exportBase') - async exportBase(@Param('baseId') baseId: string, @Body() body: any) { - return await this.exportImportService.exportBase({ - baseId, - path: body.path, - }); - } - - @Post('/api/v1/db/meta/import/:projectId/:baseId') - @HttpCode(200) - @Acl('importBase') - async importBase( - @Request() req, - @Param('projectId') projectId: string, - @Param('baseId') baseId: string, - @Body() body: any, - ) { - return await this.exportImportService.importBase({ - user: (req as any).user, - projectId: projectId, - baseId: baseId, - src: body.src, - req, - }); - } -} diff --git a/packages/nocodb-nest/src/helpers/exportImportHelpers.ts b/packages/nocodb-nest/src/helpers/exportImportHelpers.ts index 8675a5f14a..8969ba9391 100644 --- a/packages/nocodb-nest/src/helpers/exportImportHelpers.ts +++ b/packages/nocodb-nest/src/helpers/exportImportHelpers.ts @@ -70,3 +70,13 @@ export function findWithIdentifier(map: Map, id: string) { } return undefined; } + +export function generateUniqueName(name: string, names: string[]) { + let newName = name; + let i = 1; + while (names.includes(newName)) { + newName = `${name}_${i}`; + i++; + } + return newName; +} diff --git a/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts new file mode 100644 index 0000000000..a27ffe6ff1 --- /dev/null +++ b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts @@ -0,0 +1,42 @@ +import { InjectQueue } from '@nestjs/bull'; +import { + Body, + Controller, + HttpCode, + Param, + Post, + Request, + UseGuards, +} from '@nestjs/common'; +import { Queue } from 'bull'; +import { GlobalGuard } from 'src/guards/global/global.guard'; +import { + Acl, + ExtractProjectIdMiddleware, +} from 'src/middlewares/extract-project-id/extract-project-id.middleware'; + +@Controller() +@UseGuards(ExtractProjectIdMiddleware, GlobalGuard) +export class DuplicateController { + constructor( + @InjectQueue('duplicate') private readonly duplicateQueue: Queue, + ) {} + + @Post('/api/v1/db/meta/duplicate/:projectId/:baseId') + @HttpCode(200) + @Acl('duplicateBase') + async duplicateBase( + @Request() req, + @Param('projectId') projectId: string, + @Param('baseId') baseId?: string, + ) { + await this.duplicateQueue.add('duplicate', { + projectId, + baseId, + req: { + user: req.user, + clientIp: req.clientIp, + }, + }); + } +} diff --git a/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts new file mode 100644 index 0000000000..412552b519 --- /dev/null +++ b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts @@ -0,0 +1,102 @@ +import { + OnQueueActive, + OnQueueCompleted, + OnQueueFailed, + Process, + Processor, +} from '@nestjs/bull'; +import { Base, Project } from 'src/models'; +import { Job } from 'bull'; +import { ProjectsService } from 'src/services/projects.service'; +import boxen from 'boxen'; +import { generateUniqueName } from 'src/helpers/exportImportHelpers'; +import { ExportService } from './export.service'; +import { ImportService } from './import.service'; + +@Processor('duplicate') +export class DuplicateProcessor { + constructor( + private exportService: ExportService, + private importService: ImportService, + private projectsService: ProjectsService, + ) {} + + @OnQueueActive() + onActive(job: Job) { + console.log( + `Processing job ${job.id} of type ${job.name} with data ${job.data}...`, + ); + } + + @OnQueueFailed() + onFailed(job: Job, error: Error) { + console.error( + boxen( + `---- !! JOB FAILED !! ----\ntype: ${job.name}\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, + { + padding: 1, + borderStyle: 'double', + borderColor: 'yellow', + }, + ), + ); + } + + @OnQueueCompleted() + onCompleted(job: Job) { + console.log(`Completed job ${job.id} of type ${job.name}!`); + } + + @Process('duplicate') + async duplicateBase(job: Job) { + const param: { projectId: string; baseId?: string; req: any } = job.data; + + const user = (param.req as any).user; + + const project = await Project.get(param.projectId); + + if (!project) { + throw new Error(`Base not found for id '${param.baseId}'`); + } + + const base = param?.baseId + ? await Base.get(param.baseId) + : (await project.getBases())[0]; + + if (!base) { + throw new Error(`Base not found!`); + } + + const exported = await this.exportService.exportBase({ + path: `${job.name}_${job.id}`, + baseId: base.id, + }); + + if (!exported) { + throw new Error(`Export failed for base '${base.id}'`); + } + + const projects = await Project.list({}); + + const uniqueTitle = generateUniqueName( + `${project.title} copy`, + projects.map((p) => p.title), + ); + + const dupProject = await this.projectsService.projectCreate({ + project: { title: uniqueTitle }, + user: { id: user.id }, + }); + + await this.importService.importBase({ + user, + projectId: dupProject.id, + baseId: dupProject.bases[0].id, + src: { + type: 'local', + path: exported.path, + }, + req: param.req, + }); + } +} diff --git a/packages/nocodb-nest/src/modules/jobs/export-import/export.service.ts b/packages/nocodb-nest/src/modules/jobs/export-import/export.service.ts new file mode 100644 index 0000000000..29d1687f2c --- /dev/null +++ b/packages/nocodb-nest/src/modules/jobs/export-import/export.service.ts @@ -0,0 +1,487 @@ +import { Readable } from 'stream'; +import { UITypes, ViewTypes } from 'nocodb-sdk'; +import { getViewAndModelByAliasOrId } from 'src/modules/datas/helpers'; +import { unparse } from 'papaparse'; +import { + clearPrefix, + generateBaseIdMap, +} from 'src/helpers/exportImportHelpers'; +import NcPluginMgrv2 from 'src/helpers/NcPluginMgrv2'; +import { NcError } from 'src/helpers/catchError'; +import { Base, Model, Project } from 'src/models'; +import { DatasService } from 'src/services/datas.service'; +import { Injectable } from '@nestjs/common'; +import type { LinkToAnotherRecordColumn, View } from 'src/models'; +import type { IStorageAdapterV2 } from 'nc-plugin'; + +@Injectable() +export class ExportService { + constructor(private datasService: DatasService) {} + + async serializeModels(param: { modelIds: string[] }) { + const { modelIds } = param; + + const serializedModels = []; + + // db id to structured id + const idMap = new Map(); + + const projects: Project[] = []; + const bases: Base[] = []; + const modelsMap = new Map(); + + for (const modelId of modelIds) { + const model = await Model.get(modelId); + + if (!model) + return NcError.badRequest(`Model not found for id '${modelId}'`); + + const fndProject = projects.find((p) => p.id === model.project_id); + const project = fndProject || (await Project.get(model.project_id)); + + const fndBase = bases.find((b) => b.id === model.base_id); + const base = fndBase || (await Base.get(model.base_id)); + + if (!fndProject) projects.push(project); + if (!fndBase) bases.push(base); + + if (!modelsMap.has(base.id)) { + modelsMap.set(base.id, await generateBaseIdMap(base, idMap)); + } + + await model.getColumns(); + await model.getViews(); + + for (const column of model.columns) { + await column.getColOptions(); + if (column.colOptions) { + for (const [k, v] of Object.entries(column.colOptions)) { + switch (k) { + case 'fk_mm_child_column_id': + case 'fk_mm_parent_column_id': + case 'fk_mm_model_id': + case 'fk_parent_column_id': + case 'fk_child_column_id': + case 'fk_related_model_id': + case 'fk_relation_column_id': + case 'fk_lookup_column_id': + case 'fk_rollup_column_id': + column.colOptions[k] = idMap.get(v as string); + break; + case 'options': + for (const o of column.colOptions['options']) { + delete o.id; + delete o.fk_column_id; + } + break; + case 'formula': + column.colOptions[k] = column.colOptions[k].replace( + /(?<=\{\{).*?(?=\}\})/gm, + (match) => idMap.get(match), + ); + break; + case 'id': + case 'created_at': + case 'updated_at': + case 'fk_column_id': + delete column.colOptions[k]; + break; + } + } + } + } + + for (const view of model.views) { + idMap.set(view.id, `${idMap.get(model.id)}::${view.id}`); + await view.getColumns(); + await view.getFilters(); + await view.getSorts(); + if (view.filter) { + const export_filters = []; + for (const fl of view.filter.children) { + const tempFl = { + id: `${idMap.get(view.id)}::${fl.id}`, + fk_column_id: idMap.get(fl.fk_column_id), + fk_parent_id: fl.fk_parent_id, + is_group: fl.is_group, + logical_op: fl.logical_op, + comparison_op: fl.comparison_op, + comparison_sub_op: fl.comparison_sub_op, + value: fl.value, + }; + if (tempFl.is_group) { + delete tempFl.comparison_op; + delete tempFl.comparison_sub_op; + delete tempFl.value; + } + export_filters.push(tempFl); + } + view.filter.children = export_filters; + } + + if (view.sorts) { + const export_sorts = []; + for (const sr of view.sorts) { + const tempSr = { + fk_column_id: idMap.get(sr.fk_column_id), + direction: sr.direction, + }; + export_sorts.push(tempSr); + } + view.sorts = export_sorts; + } + + if (view.view) { + for (const [k, v] of Object.entries(view.view)) { + switch (k) { + case 'fk_column_id': + case 'fk_cover_image_col_id': + case 'fk_grp_col_id': + view.view[k] = idMap.get(v as string); + break; + case 'meta': + if (view.type === ViewTypes.KANBAN) { + const meta = JSON.parse(view.view.meta as string) as Record< + string, + any + >; + for (const [k, v] of Object.entries(meta)) { + const colId = idMap.get(k as string); + for (const op of v) { + op.fk_column_id = idMap.get(op.fk_column_id); + delete op.id; + } + meta[colId] = v; + delete meta[k]; + } + view.view.meta = meta; + } + break; + case 'created_at': + case 'updated_at': + case 'fk_view_id': + case 'project_id': + case 'base_id': + case 'uuid': + delete view.view[k]; + break; + } + } + } + } + + serializedModels.push({ + entity: 'model', + model: { + id: idMap.get(model.id), + prefix: project.prefix, + title: model.title, + table_name: clearPrefix(model.table_name, project.prefix), + meta: model.meta, + columns: model.columns.map((column) => ({ + id: idMap.get(column.id), + ai: column.ai, + column_name: column.column_name, + cc: column.cc, + cdf: column.cdf, + meta: column.meta, + pk: column.pk, + order: column.order, + rqd: column.rqd, + system: column.system, + uidt: column.uidt, + title: column.title, + un: column.un, + unique: column.unique, + colOptions: column.colOptions, + })), + }, + views: model.views.map((view) => ({ + id: idMap.get(view.id), + is_default: view.is_default, + type: view.type, + meta: view.meta, + order: view.order, + title: view.title, + show: view.show, + show_system_fields: view.show_system_fields, + filter: view.filter, + sorts: view.sorts, + lock_type: view.lock_type, + columns: view.columns.map((column) => { + const { + id, + fk_view_id, + fk_column_id, + project_id, + base_id, + created_at, + updated_at, + uuid, + ...rest + } = column as any; + return { + fk_column_id: idMap.get(fk_column_id), + ...rest, + }; + }), + view: view.view, + })), + }); + } + + return serializedModels; + } + + async exportModelData(param: { + storageAdapter: IStorageAdapterV2; + path: string; + projectId: string; + modelId: string; + viewId?: string; + }) { + const { model, view } = await getViewAndModelByAliasOrId({ + projectName: param.projectId, + tableName: param.modelId, + viewName: param.viewId, + }); + + await model.getColumns(); + + const hasLink = model.columns.some( + (c) => + c.uidt === UITypes.LinkToAnotherRecord && c.colOptions?.type === 'mm', + ); + + const pkMap = new Map(); + + for (const column of model.columns.filter( + (c) => + c.uidt === UITypes.LinkToAnotherRecord && c.colOptions?.type !== 'hm', + )) { + const relatedTable = await ( + (await column.getColOptions()) as LinkToAnotherRecordColumn + ).getRelatedTable(); + + await relatedTable.getColumns(); + + pkMap.set(column.id, relatedTable.primaryKey.title); + } + + const readableStream = new Readable({ + read() {}, + }); + + const readableLinkStream = new Readable({ + read() {}, + }); + + readableStream.setEncoding('utf8'); + + readableLinkStream.setEncoding('utf8'); + + const storageAdapter = param.storageAdapter; + + const uploadPromise = storageAdapter.fileCreateByStream( + `${param.path}/${model.id}.csv`, + readableStream, + ); + + const uploadLinkPromise = hasLink + ? storageAdapter.fileCreateByStream( + `${param.path}/${model.id}_links.csv`, + readableLinkStream, + ) + : Promise.resolve(); + + const limit = 200; + const offset = 0; + + const primaryKey = model.columns.find((c) => c.pk); + + const formatData = (data: any) => { + const linkData = []; + for (const row of data) { + const pkValue = primaryKey ? row[primaryKey.title] : undefined; + const linkRow = {}; + for (const [k, v] of Object.entries(row)) { + const col = model.columns.find((c) => c.title === k); + if (col) { + if (col.pk) linkRow['pk'] = pkValue; + const colId = `${col.project_id}::${col.base_id}::${col.fk_model_id}::${col.id}`; + switch (col.uidt) { + case UITypes.LinkToAnotherRecord: + { + if (col.system || col.colOptions.type === 'hm') break; + + const pkList = []; + const links = Array.isArray(v) ? v : [v]; + + for (const link of links) { + if (link) { + for (const [k, val] of Object.entries(link)) { + if (k === pkMap.get(col.id)) { + pkList.push(val); + } + } + } + } + + if (col.colOptions.type === 'mm') { + linkRow[colId] = pkList.join(','); + } else { + row[colId] = pkList[0]; + } + } + break; + case UITypes.Attachment: + try { + row[colId] = JSON.stringify(v); + } catch (e) { + row[colId] = v; + } + break; + case UITypes.ForeignKey: + case UITypes.Formula: + case UITypes.Lookup: + case UITypes.Rollup: + case UITypes.Rating: + case UITypes.Barcode: + // skip these types + break; + default: + row[colId] = v; + break; + } + delete row[k]; + } + } + linkData.push(linkRow); + } + return { data, linkData }; + }; + + try { + await this.recursiveRead( + formatData, + readableStream, + readableLinkStream, + model, + view, + offset, + limit, + true, + ); + await uploadPromise; + await uploadLinkPromise; + } catch (e) { + await storageAdapter.fileDelete(`${param.path}/${model.id}.csv`); + await storageAdapter.fileDelete(`${param.path}/${model.id}_links.csv`); + console.error(e); + throw e; + } + } + + async recursiveRead( + formatter: (data: any) => { data: any; linkData: any }, + stream: Readable, + linkStream: Readable, + model: Model, + view: View, + offset: number, + limit: number, + header = false, + ): Promise { + return new Promise((resolve, reject) => { + this.datasService + .getDataList({ model, view, query: { limit, offset } }) + .then((result) => { + try { + if (!header) { + stream.push('\r\n'); + linkStream.push('\r\n'); + } + const { data, linkData } = formatter(result.list); + stream.push(unparse(data, { header })); + linkStream.push(unparse(linkData, { header })); + if (result.pageInfo.isLastPage) { + stream.push(null); + linkStream.push(null); + resolve(); + } else { + this.recursiveRead( + formatter, + stream, + linkStream, + model, + view, + offset + limit, + limit, + ).then(resolve); + } + } catch (e) { + reject(e); + } + }); + }); + } + + async exportBase(param: { path: string; baseId: string }) { + const base = await Base.get(param.baseId); + + if (!base) + throw NcError.badRequest(`Base not found for id '${param.baseId}'`); + + const project = await Project.get(base.project_id); + + const models = (await base.getModels()).filter( + (m) => !m.mm && m.type === 'table', + ); + + const exportedModels = await this.serializeModels({ + modelIds: models.map((m) => m.id), + }); + + const exportData = { + id: `${project.id}::${base.id}`, + entity: 'base', + models: exportedModels, + }; + + const storageAdapter = await NcPluginMgrv2.storageAdapter(); + + const destPath = `export/${project.id}/${base.id}/${param.path}`; + + try { + const readableStream = new Readable({ + read() {}, + }); + + readableStream.setEncoding('utf8'); + + readableStream.push(JSON.stringify(exportData)); + + readableStream.push(null); + + await storageAdapter.fileCreateByStream( + `${destPath}/schema.json`, + readableStream, + ); + + for (const model of models) { + await this.exportModelData({ + storageAdapter, + path: `${destPath}/data`, + projectId: project.id, + modelId: model.id, + }); + } + } catch (e) { + throw NcError.badRequest(e); + } + + return { + path: destPath, + }; + } +} diff --git a/packages/nocodb-nest/src/services/export-import.service.ts b/packages/nocodb-nest/src/modules/jobs/export-import/import.service.ts similarity index 65% rename from packages/nocodb-nest/src/services/export-import.service.ts rename to packages/nocodb-nest/src/modules/jobs/export-import/import.service.ts index 46d6d9b065..80aad94e6a 100644 --- a/packages/nocodb-nest/src/services/export-import.service.ts +++ b/packages/nocodb-nest/src/modules/jobs/export-import/import.service.ts @@ -1,41 +1,34 @@ -import { Readable } from 'stream'; -import { Injectable } from '@nestjs/common'; import { UITypes, ViewTypes } from 'nocodb-sdk'; -import { getViewAndModelByAliasOrId } from 'src/modules/datas/helpers'; -import papaparse, { unparse } from 'papaparse'; import { - clearPrefix, findWithIdentifier, - generateBaseIdMap, getParentIdentifier, reverseGet, withoutId, withoutNull, } from 'src/helpers/exportImportHelpers'; -import NcPluginMgrv2 from 'src/helpers/NcPluginMgrv2'; import { NcError } from 'src/helpers/catchError'; import { Base, Column, Model, Project } from 'src/models'; -import { DatasService } from './datas.service'; -import { TablesService } from './tables.service'; -import { ColumnsService } from './columns.service'; -import { FiltersService } from './filters.service'; -import { SortsService } from './sorts.service'; -import { ViewColumnsService } from './view-columns.service'; -import { GridColumnsService } from './grid-columns.service'; -import { FormColumnsService } from './form-columns.service'; -import { GridsService } from './grids.service'; -import { FormsService } from './forms.service'; -import { GalleriesService } from './galleries.service'; -import { KanbansService } from './kanbans.service'; -import { BulkDataAliasService } from './bulk-data-alias.service'; +import { TablesService } from 'src/services/tables.service'; +import { ColumnsService } from 'src/services/columns.service'; +import { FiltersService } from 'src/services/filters.service'; +import { SortsService } from 'src/services/sorts.service'; +import { ViewColumnsService } from 'src/services/view-columns.service'; +import { GridColumnsService } from 'src/services/grid-columns.service'; +import { FormColumnsService } from 'src/services/form-columns.service'; +import { GridsService } from 'src/services/grids.service'; +import { FormsService } from 'src/services/forms.service'; +import { GalleriesService } from 'src/services/galleries.service'; +import { KanbansService } from 'src/services/kanbans.service'; +import { Injectable } from '@nestjs/common'; +import NcPluginMgrv2 from 'src/helpers/NcPluginMgrv2'; +import papaparse from 'papaparse'; +import { BulkDataAliasService } from 'src/services/bulk-data-alias.service'; import type { ViewCreateReqType } from 'nocodb-sdk'; import type { LinkToAnotherRecordColumn, User, View } from 'src/models'; -import type { IStorageAdapterV2 } from 'nc-plugin'; @Injectable() -export class ExportImportService { +export class ImportService { constructor( - private datasService: DatasService, private tablesService: TablesService, private columnsService: ColumnsService, private filtersService: FiltersService, @@ -47,499 +40,9 @@ export class ExportImportService { private formsService: FormsService, private galleriesService: GalleriesService, private kanbansService: KanbansService, - private bulkDatasService: BulkDataAliasService, + private bulkDataService: BulkDataAliasService, ) {} - async serializeModels(param: { modelId: string[] }) { - const serializedModels = []; - - // db id to structured id - const idMap = new Map(); - - const projects: Project[] = []; - const bases: Base[] = []; - const modelsMap = new Map(); - - for (const modelId of param.modelId) { - const model = await Model.get(modelId); - - if (!model) - return NcError.badRequest(`Model not found for id '${modelId}'`); - - const fndProject = projects.find((p) => p.id === model.project_id); - const project = fndProject || (await Project.get(model.project_id)); - - const fndBase = bases.find((b) => b.id === model.base_id); - const base = fndBase || (await Base.get(model.base_id)); - - if (!fndProject) projects.push(project); - if (!fndBase) bases.push(base); - - if (!modelsMap.has(base.id)) { - modelsMap.set(base.id, await generateBaseIdMap(base, idMap)); - } - - await model.getColumns(); - await model.getViews(); - - for (const column of model.columns) { - await column.getColOptions(); - if (column.colOptions) { - for (const [k, v] of Object.entries(column.colOptions)) { - switch (k) { - case 'fk_mm_child_column_id': - case 'fk_mm_parent_column_id': - case 'fk_mm_model_id': - case 'fk_parent_column_id': - case 'fk_child_column_id': - case 'fk_related_model_id': - case 'fk_relation_column_id': - case 'fk_lookup_column_id': - case 'fk_rollup_column_id': - column.colOptions[k] = idMap.get(v as string); - break; - case 'options': - for (const o of column.colOptions['options']) { - delete o.id; - delete o.fk_column_id; - } - break; - case 'formula': - column.colOptions[k] = column.colOptions[k].replace( - /(?<=\{\{).*?(?=\}\})/gm, - (match) => idMap.get(match), - ); - break; - case 'id': - case 'created_at': - case 'updated_at': - case 'fk_column_id': - delete column.colOptions[k]; - break; - } - } - } - } - - for (const view of model.views) { - idMap.set(view.id, `${idMap.get(model.id)}::${view.id}`); - await view.getColumns(); - await view.getFilters(); - await view.getSorts(); - if (view.filter) { - const export_filters = []; - for (const fl of view.filter.children) { - const tempFl = { - id: `${idMap.get(view.id)}::${fl.id}`, - fk_column_id: idMap.get(fl.fk_column_id), - fk_parent_id: fl.fk_parent_id, - is_group: fl.is_group, - logical_op: fl.logical_op, - comparison_op: fl.comparison_op, - comparison_sub_op: fl.comparison_sub_op, - value: fl.value, - }; - if (tempFl.is_group) { - delete tempFl.comparison_op; - delete tempFl.comparison_sub_op; - delete tempFl.value; - } - export_filters.push(tempFl); - } - view.filter.children = export_filters; - } - - if (view.sorts) { - const export_sorts = []; - for (const sr of view.sorts) { - const tempSr = { - fk_column_id: idMap.get(sr.fk_column_id), - direction: sr.direction, - }; - export_sorts.push(tempSr); - } - view.sorts = export_sorts; - } - - if (view.view) { - for (const [k, v] of Object.entries(view.view)) { - switch (k) { - case 'fk_column_id': - case 'fk_cover_image_col_id': - case 'fk_grp_col_id': - view.view[k] = idMap.get(v as string); - break; - case 'meta': - if (view.type === ViewTypes.KANBAN) { - const meta = JSON.parse(view.view.meta as string) as Record< - string, - any - >; - for (const [k, v] of Object.entries(meta)) { - const colId = idMap.get(k as string); - for (const op of v) { - op.fk_column_id = idMap.get(op.fk_column_id); - delete op.id; - } - meta[colId] = v; - delete meta[k]; - } - view.view.meta = meta; - } - break; - case 'created_at': - case 'updated_at': - case 'fk_view_id': - case 'project_id': - case 'base_id': - case 'uuid': - delete view.view[k]; - break; - } - } - } - } - - serializedModels.push({ - entity: 'model', - model: { - id: idMap.get(model.id), - prefix: project.prefix, - title: model.title, - table_name: clearPrefix(model.table_name, project.prefix), - meta: model.meta, - columns: model.columns.map((column) => ({ - id: idMap.get(column.id), - ai: column.ai, - column_name: column.column_name, - cc: column.cc, - cdf: column.cdf, - meta: column.meta, - pk: column.pk, - order: column.order, - rqd: column.rqd, - system: column.system, - uidt: column.uidt, - title: column.title, - un: column.un, - unique: column.unique, - colOptions: column.colOptions, - })), - }, - views: model.views.map((view) => ({ - id: idMap.get(view.id), - is_default: view.is_default, - type: view.type, - meta: view.meta, - order: view.order, - title: view.title, - show: view.show, - show_system_fields: view.show_system_fields, - filter: view.filter, - sorts: view.sorts, - lock_type: view.lock_type, - columns: view.columns.map((column) => { - const { - id, - fk_view_id, - fk_column_id, - project_id, - base_id, - created_at, - updated_at, - uuid, - ...rest - } = column as any; - return { - fk_column_id: idMap.get(fk_column_id), - ...rest, - }; - }), - view: view.view, - })), - }); - } - - return serializedModels; - } - - async exportModelData(param: { - storageAdapter: IStorageAdapterV2; - path: string; - projectId: string; - modelId: string; - viewId?: string; - }) { - const { model, view } = await getViewAndModelByAliasOrId({ - projectName: param.projectId, - tableName: param.modelId, - viewName: param.viewId, - }); - - await model.getColumns(); - - const hasLink = model.columns.some( - (c) => - c.uidt === UITypes.LinkToAnotherRecord && c.colOptions?.type === 'mm', - ); - - const pkMap = new Map(); - - for (const column of model.columns.filter( - (c) => - c.uidt === UITypes.LinkToAnotherRecord && c.colOptions?.type !== 'hm', - )) { - const relatedTable = await ( - (await column.getColOptions()) as LinkToAnotherRecordColumn - ).getRelatedTable(); - - await relatedTable.getColumns(); - - pkMap.set(column.id, relatedTable.primaryKey.title); - } - - const readableStream = new Readable({ - read() {}, - }); - - const readableLinkStream = new Readable({ - read() {}, - }); - - readableStream.setEncoding('utf8'); - - readableLinkStream.setEncoding('utf8'); - - const storageAdapter = param.storageAdapter; - - const uploadPromise = storageAdapter.fileCreateByStream( - `${param.path}/${model.id}.csv`, - readableStream, - ); - - const uploadLinkPromise = hasLink - ? storageAdapter.fileCreateByStream( - `${param.path}/${model.id}_links.csv`, - readableLinkStream, - ) - : Promise.resolve(); - - const limit = 100; - const offset = 0; - - const primaryKey = model.columns.find((c) => c.pk); - - const formatData = (data: any) => { - const linkData = []; - for (const row of data) { - const pkValue = primaryKey ? row[primaryKey.title] : undefined; - const linkRow = {}; - for (const [k, v] of Object.entries(row)) { - const col = model.columns.find((c) => c.title === k); - if (col) { - if (col.pk) linkRow['pk'] = pkValue; - const colId = `${col.project_id}::${col.base_id}::${col.fk_model_id}::${col.id}`; - switch (col.uidt) { - case UITypes.LinkToAnotherRecord: - { - if (col.system || col.colOptions.type === 'hm') break; - - const pkList = []; - const links = Array.isArray(v) ? v : [v]; - - for (const link of links) { - if (link) { - for (const [k, val] of Object.entries(link)) { - if (k === pkMap.get(col.id)) { - pkList.push(val); - } - } - } - } - - if (col.colOptions.type === 'mm') { - linkRow[colId] = pkList.join(','); - } else { - row[colId] = pkList[0]; - } - } - break; - case UITypes.Attachment: - try { - row[colId] = JSON.stringify(v); - } catch (e) { - row[colId] = v; - } - break; - case UITypes.ForeignKey: - case UITypes.Formula: - case UITypes.Lookup: - case UITypes.Rollup: - case UITypes.Rating: - case UITypes.Barcode: - // skip these types - break; - default: - row[colId] = v; - break; - } - delete row[k]; - } - } - linkData.push(linkRow); - } - return { data, linkData }; - }; - - try { - await this.recursiveRead( - formatData, - readableStream, - readableLinkStream, - model, - view, - offset, - limit, - true, - ); - await uploadPromise; - await uploadLinkPromise; - } catch (e) { - await storageAdapter.fileDelete(`${param.path}/${model.id}.csv`); - await storageAdapter.fileDelete(`${param.path}/${model.id}_links.csv`); - console.error(e); - throw e; - } - - return true; - } - - async recursiveRead( - formatter: (data: any) => { data: any; linkData: any }, - stream: Readable, - linkStream: Readable, - model: Model, - view: View, - offset: number, - limit: number, - header = false, - ): Promise { - return new Promise((resolve, reject) => { - this.datasService - .getDataList({ model, view, query: { limit, offset } }) - .then((result) => { - try { - if (!header) { - stream.push('\r\n'); - linkStream.push('\r\n'); - } - const { data, linkData } = formatter(result.list); - stream.push(unparse(data, { header })); - linkStream.push(unparse(linkData, { header })); - if (result.pageInfo.isLastPage) { - stream.push(null); - linkStream.push(null); - resolve(); - } else { - this.recursiveRead( - formatter, - stream, - linkStream, - model, - view, - offset + limit, - limit, - ).then(resolve); - } - } catch (e) { - reject(e); - } - }); - }); - } - - /* - async exportBaseSchema(param: { baseId: string }) { - const base = await Base.get(param.baseId); - - if (!base) - return NcError.badRequest(`Base not found for id '${param.baseId}'`); - - const project = await Project.get(base.project_id); - - const models = (await base.getModels()).filter( - (m) => !m.mm && m.type === 'table', - ); - - const exportedModels = await this.serializeModels({ - modelId: models.map((m) => m.id), - }); - - const exportData = { - id: `${project.id}::${base.id}`, - entity: 'base', - models: exportedModels, - }; - - return exportData; - } - */ - - async exportBase(param: { path: string; baseId: string }) { - const base = await Base.get(param.baseId); - - if (!base) - return NcError.badRequest(`Base not found for id '${param.baseId}'`); - - const project = await Project.get(base.project_id); - - const models = (await base.getModels()).filter( - (m) => !m.mm && m.type === 'table', - ); - - const exportedModels = await this.serializeModels({ - modelId: models.map((m) => m.id), - }); - - const exportData = { - id: `${project.id}::${base.id}`, - entity: 'base', - models: exportedModels, - }; - - const storageAdapter = await NcPluginMgrv2.storageAdapter(); - - const destPath = `export/${project.id}/${base.id}/${param.path}/schema.json`; - - try { - const readableStream = new Readable({ - read() {}, - }); - - readableStream.setEncoding('utf8'); - - readableStream.push(JSON.stringify(exportData)); - - readableStream.push(null); - - await storageAdapter.fileCreateByStream(destPath, readableStream); - - for (const model of models) { - await this.exportModelData({ - storageAdapter, - path: `export/${project.id}/${base.id}/${param.path}/data`, - projectId: project.id, - modelId: model.id, - }); - } - } catch (e) { - console.error(e); - return NcError.internalServerError('Error while exporting base'); - } - - return true; - } - async importModels(param: { user: User; projectId: string; @@ -1153,10 +656,11 @@ export class ExportImportService { file?: any; }; req: any; + debug?: boolean; }) { const { user, projectId, baseId, src, req } = param; - const debug = req.params.debug === 'true'; + const debug = param?.debug === true; const debugLog = (...args: any[]) => { if (!debug) return; @@ -1227,7 +731,7 @@ export class ExportImportService { await new Promise((resolve) => { papaparse.parse(readStream, { newline: '\r\n', - step: async function (results, parser) { + step: async (results, parser) => { if (!headers.length) { parser.pause(); for (const header of results.data) { @@ -1264,7 +768,7 @@ export class ExportImportService { parser.pause(); elapsedTime('before import chunk'); try { - await this.bulkDatasService.bulkDataInsert({ + await this.bulkDataService.bulkDataInsert({ projectName: projectId, tableName: modelId, body: chunk, @@ -1284,11 +788,11 @@ export class ExportImportService { } } }, - complete: async function () { + complete: async () => { if (chunk.length > 0) { elapsedTime('before import chunk'); try { - await this.bulkDatasService.bulkDataInsert({ + await this.bulkDataService.bulkDataInsert({ projectName: projectId, tableName: modelId, body: chunk, @@ -1335,7 +839,7 @@ export class ExportImportService { await new Promise((resolve) => { papaparse.parse(readStream, { newline: '\r\n', - step: async function (results, parser) { + step: async (results, parser) => { if (!headers.length) { parser.pause(); for (const header of results.data) { @@ -1403,11 +907,11 @@ export class ExportImportService { } } }, - complete: async function () { + complete: async () => { for (const [k, v] of Object.entries(chunk)) { try { elapsedTime('prepare link chunk'); - await this.bulkDatasService.bulkDataInsert({ + await this.bulkDataService.bulkDataInsert({ projectName: projectId, tableName: k, body: v, diff --git a/packages/nocodb-nest/src/modules/jobs/jobs.module.ts b/packages/nocodb-nest/src/modules/jobs/jobs.module.ts new file mode 100644 index 0000000000..ae7730bf40 --- /dev/null +++ b/packages/nocodb-nest/src/modules/jobs/jobs.module.ts @@ -0,0 +1,23 @@ +import { Module } from '@nestjs/common'; +import { BullModule } from '@nestjs/bull'; +import { GlobalModule } from '../global/global.module'; +import { DatasModule } from '../datas/datas.module'; +import { MetasModule } from '../metas/metas.module'; +import { ExportService } from './export-import/export.service'; +import { ImportService } from './export-import/import.service'; +import { DuplicateController } from './export-import/duplicate.controller'; +import { DuplicateProcessor } from './export-import/duplicate.processor'; + +@Module({ + imports: [ + GlobalModule, + DatasModule, + MetasModule, + BullModule.registerQueue({ + name: 'duplicate', + }), + ], + controllers: [DuplicateController], + providers: [DuplicateProcessor, ExportService, ImportService], +}) +export class JobsModule {} diff --git a/packages/nocodb-nest/src/modules/metas/metas.module.ts b/packages/nocodb-nest/src/modules/metas/metas.module.ts index 3d8a46b4d5..1753b278e3 100644 --- a/packages/nocodb-nest/src/modules/metas/metas.module.ts +++ b/packages/nocodb-nest/src/modules/metas/metas.module.ts @@ -1,8 +1,6 @@ import { Module } from '@nestjs/common'; import { MulterModule } from '@nestjs/platform-express'; import multer from 'multer'; -import { ExportImportController } from 'src/controllers/export-import.controller'; -import { ExportImportService } from 'src/services/export-import.service'; import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants'; import { ApiDocsController } from '../../controllers/api-docs/api-docs.controller'; import { ApiTokensController } from '../../controllers/api-tokens.controller'; @@ -84,7 +82,6 @@ import { DatasModule } from '../datas/datas.module'; }, }), GlobalModule, - DatasModule, ], controllers: [ ApiDocsController, @@ -94,7 +91,6 @@ import { DatasModule } from '../datas/datas.module'; BasesController, CachesController, ColumnsController, - ExportImportController, FiltersController, FormColumnsController, FormsController, @@ -130,7 +126,6 @@ import { DatasModule } from '../datas/datas.module'; BasesService, CachesService, ColumnsService, - ExportImportService, FiltersService, FormColumnsService, FormsService, @@ -161,5 +156,20 @@ import { DatasModule } from '../datas/datas.module'; SharedBasesService, BulkDataAliasService, ], + exports: [ + TablesService, + ColumnsService, + FiltersService, + SortsService, + ViewsService, + ViewColumnsService, + GridsService, + GridColumnsService, + FormsService, + FormColumnsService, + GalleriesService, + KanbansService, + ProjectsService, + ], }) export class MetasModule {} diff --git a/packages/nocodb-nest/src/services/export-import.service.spec.ts b/packages/nocodb-nest/src/services/export-import.service.spec.ts deleted file mode 100644 index 9a20c0ac7c..0000000000 --- a/packages/nocodb-nest/src/services/export-import.service.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { ExportImportService } from './export-import.service'; -import type { TestingModule } from '@nestjs/testing'; - -describe('ExportImportService', () => { - let service: ExportImportService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ExportImportService], - }).compile(); - - service = module.get(ExportImportService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -});