From 09e9e525f6788c27ba7b32de5f45fe4372876f6a Mon Sep 17 00:00:00 2001 From: mertmit Date: Sat, 22 Apr 2023 12:03:16 +0300 Subject: [PATCH] feat: improved duplicate logic Signed-off-by: mertmit --- packages/nc-gui/pages/index/index/index.vue | 38 +- .../export-import/duplicate.controller.ts | 36 +- .../jobs/export-import/duplicate.processor.ts | 472 +++++++++--------- 3 files changed, 286 insertions(+), 260 deletions(-) diff --git a/packages/nc-gui/pages/index/index/index.vue b/packages/nc-gui/pages/index/index/index.vue index 250de32f12..d1175baee6 100644 --- a/packages/nc-gui/pages/index/index/index.vue +++ b/packages/nc-gui/pages/index/index/index.vue @@ -39,9 +39,17 @@ const filterQuery = ref('') const projects = ref() +const activePage = ref(1) + +const pageChange = (p: number) => { + activePage.value = p +} + const loadProjects = async () => { + const lastPage = activePage.value const response = await api.project.list({}) projects.value = response.list + activePage.value = lastPage } const filteredProjects = computed( @@ -85,12 +93,14 @@ const duplicateProject = (project: ProjectType) => { try { const jobData = await api.project.duplicate(project.id as string) + await loadProjects() + $events.subscribe(jobData.name, jobData.id, async (data: { status: string }) => { - console.log('dataCB', data) - if (data.status === 'completed' || data.status === 'refresh') { + if (data.status === 'completed') { await loadProjects() } else if (data.status === 'failed') { message.error('Failed to duplicate project') + await loadProjects() } }) @@ -224,7 +234,7 @@ const copyProjectMeta = async () => { v-else :custom-row="customRow" :data-source="filteredProjects" - :pagination="{ position: ['bottomCenter'] }" + :pagination="{ 'position': ['bottomCenter'], 'current': activePage, 'onUpdate:current': pageChange }" :table-layout="md ? 'auto' : 'fixed'" > diff --git a/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts index d6b4dd9297..f35444a6b9 100644 --- a/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts +++ b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts @@ -14,6 +14,9 @@ import { Acl, ExtractProjectIdMiddleware, } from 'src/middlewares/extract-project-id/extract-project-id.middleware'; +import { ProjectsService } from 'src/services/projects.service'; +import { Base, Project } from 'src/models'; +import { generateUniqueName } from 'src/helpers/exportImportHelpers'; import { QueueService } from '../fallback-queue.service'; @Controller() @@ -23,6 +26,7 @@ export class DuplicateController { constructor( @InjectQueue('jobs') private readonly jobsQueue: Queue, private readonly fallbackQueueService: QueueService, + private readonly projectsService: ProjectsService, ) { this.activeQueue = process.env.NC_REDIS_URL ? this.jobsQueue @@ -37,14 +41,42 @@ export class DuplicateController { @Param('projectId') projectId: string, @Param('baseId') baseId?: string, ) { + const project = await Project.get(projectId); + + if (!project) { + throw new Error(`Project not found for id '${projectId}'`); + } + + const base = baseId + ? await Base.get(baseId) + : (await project.getBases())[0]; + + if (!base) { + throw new Error(`Base not found!`); + } + + const projects = await Project.list({}); + + const uniqueTitle = generateUniqueName( + `${project.title} copy`, + projects.map((p) => p.title), + ); + + const dupProject = await this.projectsService.projectCreate({ + project: { title: uniqueTitle, status: 'job' }, + user: { id: req.user.id }, + }); + const job = await this.activeQueue.add('duplicate', { - projectId, - baseId, + project, + base, + dupProject, req: { user: req.user, clientIp: req.clientIp, }, }); + return { id: job.id, name: job.name }; } } diff --git a/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts index 4d0f84a3e2..7b11ef7c16 100644 --- a/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts +++ b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts @@ -6,15 +6,12 @@ import { Process, Processor, } from '@nestjs/bull'; -import { Base, Column, Model, Project } from 'src/models'; +import { Column, Model } from 'src/models'; import { Job } from 'bull'; import { ProjectsService } from 'src/services/projects.service'; import boxen from 'boxen'; import papaparse from 'papaparse'; -import { - findWithIdentifier, - generateUniqueName, -} from 'src/helpers/exportImportHelpers'; +import { findWithIdentifier } from 'src/helpers/exportImportHelpers'; import { BulkDataAliasService } from 'src/services/bulk-data-alias.service'; import { UITypes } from 'nocodb-sdk'; import { forwardRef, Inject } from '@nestjs/common'; @@ -76,295 +73,274 @@ export class DuplicateProcessor { @Process('duplicate') async duplicateBase(job: Job) { - let start = process.hrtime(); - - const debugLog = function (...args: any[]) { - if (DEBUG) { - console.log(...args); - } - }; + const { project, base, dupProject, req } = job.data; - const elapsedTime = function (label?: string) { - const elapsedS = process.hrtime(start)[0].toFixed(3); - const elapsedMs = process.hrtime(start)[1] / 1000000; - if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`); - start = process.hrtime(); - }; + try { + let start = process.hrtime(); - const param: { projectId: string; baseId?: string; req: any } = job.data; + const debugLog = function (...args: any[]) { + if (DEBUG) { + console.log(...args); + } + }; - const user = (param.req as any).user; + const elapsedTime = function (label?: string) { + const elapsedS = process.hrtime(start)[0].toFixed(3); + const elapsedMs = process.hrtime(start)[1] / 1000000; + if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`); + start = process.hrtime(); + }; - const project = await Project.get(param.projectId); - - if (!project) { - throw new Error(`Base not found for id '${param.baseId}'`); - } + const user = (req as any).user; - const base = param?.baseId - ? await Base.get(param.baseId) - : (await project.getBases())[0]; - - if (!base) { - throw new Error(`Base not found!`); - } - - const models = (await base.getModels()).filter( - (m) => !m.mm && m.type === 'table', - ); + const models = (await base.getModels()).filter( + (m) => !m.mm && m.type === 'table', + ); - const exportedModels = await this.exportService.serializeModels({ - modelIds: models.map((m) => m.id), - }); - - elapsedTime('serializeModels'); - - if (!exportedModels) { - throw new Error(`Export failed for base '${base.id}'`); - } - - const projects = await Project.list({}); + const exportedModels = await this.exportService.serializeModels({ + modelIds: models.map((m) => m.id), + }); - const uniqueTitle = generateUniqueName( - `${project.title} copy`, - projects.map((p) => p.title), - ); + elapsedTime('serializeModels'); - const dupProject = await this.projectsService.projectCreate({ - project: { title: uniqueTitle, status: 'job' }, - user: { id: user.id }, - }); - - this.jobsGateway.jobStatus({ - name: job.name, - id: job.id.toString(), - status: 'refresh', - }); + if (!exportedModels) { + throw new Error(`Export failed for base '${base.id}'`); + } - const dupBaseId = dupProject.bases[0].id; + const dupBaseId = dupProject.bases[0].id; - elapsedTime('projectCreate'); + elapsedTime('projectCreate'); - const idMap = await this.importService.importModels({ - user, - projectId: dupProject.id, - baseId: dupBaseId, - data: exportedModels, - req: param.req, - }); + const idMap = await this.importService.importModels({ + user, + projectId: dupProject.id, + baseId: dupBaseId, + data: exportedModels, + req: req, + }); - elapsedTime('importModels'); + elapsedTime('importModels'); - if (!idMap) { - throw new Error(`Import failed for base '${base.id}'`); - } + if (!idMap) { + throw new Error(`Import failed for base '${base.id}'`); + } - const handledLinks = []; - const lChunk: Record = {}; // colId: { rowId, childId }[] + const handledLinks = []; + const lChunk: Record = {}; // colId: { rowId, childId }[] - for (const sourceModel of models) { - const dataStream = new Readable({ - read() {}, - }); + for (const sourceModel of models) { + const dataStream = new Readable({ + read() {}, + }); - const linkStream = new Readable({ - read() {}, - }); + const linkStream = new Readable({ + read() {}, + }); - this.exportService.streamModelData({ - dataStream, - linkStream, - projectId: project.id, - modelId: sourceModel.id, - }); + this.exportService.streamModelData({ + dataStream, + linkStream, + projectId: project.id, + modelId: sourceModel.id, + }); - const headers: string[] = []; - let chunk = []; - - const model = await Model.get(findWithIdentifier(idMap, sourceModel.id)); - - await new Promise((resolve) => { - papaparse.parse(dataStream, { - newline: '\r\n', - step: async (results, parser) => { - if (!headers.length) { - parser.pause(); - for (const header of results.data) { - const id = idMap.get(header); - if (id) { - const col = await Column.get({ - base_id: dupBaseId, - colId: id, - }); - if (col.colOptions?.type === 'bt') { - const childCol = await Column.get({ + const headers: string[] = []; + let chunk = []; + + const model = await Model.get( + findWithIdentifier(idMap, sourceModel.id), + ); + + await new Promise((resolve) => { + papaparse.parse(dataStream, { + newline: '\r\n', + step: async (results, parser) => { + if (!headers.length) { + parser.pause(); + for (const header of results.data) { + const id = idMap.get(header); + if (id) { + const col = await Column.get({ base_id: dupBaseId, - colId: col.colOptions.fk_child_column_id, + colId: id, }); - headers.push(childCol.column_name); + if (col.colOptions?.type === 'bt') { + const childCol = await Column.get({ + base_id: dupBaseId, + colId: col.colOptions.fk_child_column_id, + }); + headers.push(childCol.column_name); + } else { + headers.push(col.column_name); + } } else { - headers.push(col.column_name); + debugLog('header not found', header); } - } else { - debugLog('header not found', header); } - } - parser.resume(); - } else { - if (results.errors.length === 0) { - const row = {}; - for (let i = 0; i < headers.length; i++) { - if (results.data[i] !== '') { - row[headers[i]] = results.data[i]; + parser.resume(); + } else { + if (results.errors.length === 0) { + const row = {}; + for (let i = 0; i < headers.length; i++) { + if (results.data[i] !== '') { + row[headers[i]] = results.data[i]; + } } - } - chunk.push(row); - if (chunk.length > 1000) { - parser.pause(); - try { - await this.bulkDataService.bulkDataInsert({ - projectName: dupProject.id, - tableName: model.id, - body: chunk, - cookie: null, - chunkSize: chunk.length + 1, - foreign_key_checks: false, - raw: true, - }); - } catch (e) { - console.log(e); + chunk.push(row); + if (chunk.length > 1000) { + parser.pause(); + try { + await this.bulkDataService.bulkDataInsert({ + projectName: dupProject.id, + tableName: model.id, + body: chunk, + cookie: null, + chunkSize: chunk.length + 1, + foreign_key_checks: false, + raw: true, + }); + } catch (e) { + console.log(e); + } + chunk = []; + parser.resume(); } - chunk = []; - parser.resume(); } } - } - }, - complete: async () => { - if (chunk.length > 0) { - try { - await this.bulkDataService.bulkDataInsert({ - projectName: dupProject.id, - tableName: model.id, - body: chunk, - cookie: null, - chunkSize: chunk.length + 1, - foreign_key_checks: false, - raw: true, - }); - } catch (e) { - console.log(e); + }, + complete: async () => { + if (chunk.length > 0) { + try { + await this.bulkDataService.bulkDataInsert({ + projectName: dupProject.id, + tableName: model.id, + body: chunk, + cookie: null, + chunkSize: chunk.length + 1, + foreign_key_checks: false, + raw: true, + }); + } catch (e) { + console.log(e); + } + chunk = []; } - chunk = []; - } - resolve(null); - }, + resolve(null); + }, + }); }); - }); - const lHeaders: string[] = []; - const mmParentChild: any = {}; - - let pkIndex = -1; - - await new Promise((resolve) => { - papaparse.parse(linkStream, { - newline: '\r\n', - step: async (results, parser) => { - if (!lHeaders.length) { - parser.pause(); - for (const header of results.data) { - if (header === 'pk') { - lHeaders.push(null); - pkIndex = lHeaders.length - 1; - continue; - } - const id = idMap.get(header); - if (id) { - const col = await Column.get({ - base_id: dupBaseId, - colId: id, - }); - if ( - col.uidt === UITypes.LinkToAnotherRecord && - col.colOptions.fk_mm_model_id && - handledLinks.includes(col.colOptions.fk_mm_model_id) - ) { + const lHeaders: string[] = []; + const mmParentChild: any = {}; + + let pkIndex = -1; + + await new Promise((resolve) => { + papaparse.parse(linkStream, { + newline: '\r\n', + step: async (results, parser) => { + if (!lHeaders.length) { + parser.pause(); + for (const header of results.data) { + if (header === 'pk') { lHeaders.push(null); - } else { + pkIndex = lHeaders.length - 1; + continue; + } + const id = idMap.get(header); + if (id) { + const col = await Column.get({ + base_id: dupBaseId, + colId: id, + }); if ( col.uidt === UITypes.LinkToAnotherRecord && col.colOptions.fk_mm_model_id && - !handledLinks.includes(col.colOptions.fk_mm_model_id) + handledLinks.includes(col.colOptions.fk_mm_model_id) ) { - const colOptions = - await col.getColOptions(); - - const vChildCol = await colOptions.getMMChildColumn(); - const vParentCol = await colOptions.getMMParentColumn(); - - mmParentChild[col.colOptions.fk_mm_model_id] = { - parent: vParentCol.column_name, - child: vChildCol.column_name, - }; - - handledLinks.push(col.colOptions.fk_mm_model_id); + lHeaders.push(null); + } else { + if ( + col.uidt === UITypes.LinkToAnotherRecord && + col.colOptions.fk_mm_model_id && + !handledLinks.includes(col.colOptions.fk_mm_model_id) + ) { + const colOptions = + await col.getColOptions(); + + const vChildCol = await colOptions.getMMChildColumn(); + const vParentCol = await colOptions.getMMParentColumn(); + + mmParentChild[col.colOptions.fk_mm_model_id] = { + parent: vParentCol.column_name, + child: vChildCol.column_name, + }; + + handledLinks.push(col.colOptions.fk_mm_model_id); + } + lHeaders.push(col.colOptions.fk_mm_model_id); + lChunk[col.colOptions.fk_mm_model_id] = []; } - lHeaders.push(col.colOptions.fk_mm_model_id); - lChunk[col.colOptions.fk_mm_model_id] = []; } } - } - parser.resume(); - } else { - if (results.errors.length === 0) { - for (let i = 0; i < lHeaders.length; i++) { - if (!lHeaders[i]) continue; - - const mm = mmParentChild[lHeaders[i]]; - - for (const rel of results.data[i].split(',')) { - if (rel.trim() === '') continue; - lChunk[lHeaders[i]].push({ - [mm.parent]: rel, - [mm.child]: results.data[pkIndex], - }); + parser.resume(); + } else { + if (results.errors.length === 0) { + for (let i = 0; i < lHeaders.length; i++) { + if (!lHeaders[i]) continue; + + const mm = mmParentChild[lHeaders[i]]; + + for (const rel of results.data[i].split(',')) { + if (rel.trim() === '') continue; + lChunk[lHeaders[i]].push({ + [mm.parent]: rel, + [mm.child]: results.data[pkIndex], + }); + } } } } - } - }, - complete: async () => { - resolve(null); - }, + }, + complete: async () => { + resolve(null); + }, + }); }); - }); - elapsedTime(model.title); - } + elapsedTime(model.title); + } + + for (const [k, v] of Object.entries(lChunk)) { + try { + await this.bulkDataService.bulkDataInsert({ + projectName: dupProject.id, + tableName: k, + body: v, + cookie: null, + chunkSize: 1000, + foreign_key_checks: false, + raw: true, + }); + } catch (e) { + console.log(e); + } + } - for (const [k, v] of Object.entries(lChunk)) { - try { - await this.bulkDataService.bulkDataInsert({ - projectName: dupProject.id, - tableName: k, - body: v, - cookie: null, - chunkSize: 1000, - foreign_key_checks: false, - raw: true, + elapsedTime('links'); + await this.projectsService.projectUpdate({ + projectId: dupProject.id, + project: { + status: null, + }, + }); + } catch (e) { + if (dupProject?.id) { + await this.projectsService.projectSoftDelete({ + projectId: dupProject.id, }); - } catch (e) { - console.log(e); } + throw e; } - - elapsedTime('links'); - await this.projectsService.projectUpdate({ - projectId: dupProject.id, - project: { - status: null, - }, - }); } }