github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: improved duplicate logic 

Signed-off-by: mertmit <mertmit99@gmail.com>
feat/export-nest
mertmit 1 year ago
parent
e49700a6b3
commit
63a7b9a5c3
3 changed files with 286 additions and 260 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 38
      packages/nc-gui/pages/index/index/index.vue
  2. 36
      packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts
  3. 472
      packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts

38
packages/nc-gui/pages/index/index/index.vue
Unescape View File

@ -39,9 +39,17 @@ const filterQuery = ref('')
const projects = ref<ProjectType[]>() const projects = ref<ProjectType[]>()
const activePage = ref(1)
const pageChange = (p: number) => {
activePage.value = p
}
const loadProjects = async () => { const loadProjects = async () => {
const lastPage = activePage.value
const response = await api.project.list({}) const response = await api.project.list({})
projects.value = response.list projects.value = response.list
activePage.value = lastPage
} }
const filteredProjects = computed( const filteredProjects = computed(
@ -85,12 +93,14 @@ const duplicateProject = (project: ProjectType) => {
try { try {
const jobData = await api.project.duplicate(project.id as string) const jobData = await api.project.duplicate(project.id as string)
await loadProjects()
$events.subscribe(jobData.name, jobData.id, async (data: { status: string }) => { $events.subscribe(jobData.name, jobData.id, async (data: { status: string }) => {
console.log('dataCB', data) if (data.status === 'completed') {
if (data.status === 'completed' || data.status === 'refresh') {
await loadProjects() await loadProjects()
} else if (data.status === 'failed') { } else if (data.status === 'failed') {
message.error('Failed to duplicate project') message.error('Failed to duplicate project')
await loadProjects()
} }
}) })
@ -224,7 +234,7 @@ const copyProjectMeta = async () => {
v-else v-else
:custom-row="customRow" :custom-row="customRow"
:data-source="filteredProjects" :data-source="filteredProjects"
:pagination="{ position: ['bottomCenter'] }" :pagination="{ 'position': ['bottomCenter'], 'current': activePage, 'onUpdate:current': pageChange }"
:table-layout="md ? 'auto' : 'fixed'" :table-layout="md ? 'auto' : 'fixed'"
> >
<template #emptyText> <template #emptyText>
@ -301,19 +311,27 @@ const copyProjectMeta = async () => {
@click.stop="navigateTo(`/${text}`)" @click.stop="navigateTo(`/${text}`)"
/> />
<component
:is="iconMap.copy"
v-e="['c:project:duplicate']"
class="nc-action-btn"
@click.stop="duplicateProject(record)"
/>
<component <component
:is="iconMap.delete" :is="iconMap.delete"
class="nc-action-btn" class="nc-action-btn"
:data-testid="`delete-project-${record.title}`" :data-testid="`delete-project-${record.title}`"
@click.stop="deleteProject(record)" @click.stop="deleteProject(record)"
/> />
<a-dropdown :trigger="['click']" overlay-class-name="nc-dropdown-import-menu" @click.stop>
<GeneralIcon icon="threeDotVertical" class="nc-import-menu outline-0" />
<template #overlay>
<a-menu class="!py-0 rounded text-sm">
<a-menu-item key="duplicate" v-e="['c:project:duplicate']" @click.stop="duplicateProject(record)">
<div class="color-transition nc-project-menu-item group">
<GeneralIcon icon="copy" class="group-hover:text-accent" />
Duplicate
</div>
</a-menu-item>
</a-menu>
</template>
</a-dropdown>
</div> </div>
</template> </template>
</a-table-column> </a-table-column>

36
packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts
Unescape View File

@ -14,6 +14,9 @@ import {
Acl, Acl,
ExtractProjectIdMiddleware, ExtractProjectIdMiddleware,
} from 'src/middlewares/extract-project-id/extract-project-id.middleware'; } from 'src/middlewares/extract-project-id/extract-project-id.middleware';
import { ProjectsService } from 'src/services/projects.service';
import { Base, Project } from 'src/models';
import { generateUniqueName } from 'src/helpers/exportImportHelpers';
import { QueueService } from '../fallback-queue.service'; import { QueueService } from '../fallback-queue.service';
@Controller() @Controller()
@ -23,6 +26,7 @@ export class DuplicateController {
constructor( constructor(
@InjectQueue('jobs') private readonly jobsQueue: Queue, @InjectQueue('jobs') private readonly jobsQueue: Queue,
private readonly fallbackQueueService: QueueService, private readonly fallbackQueueService: QueueService,
private readonly projectsService: ProjectsService,
) { ) {
this.activeQueue = process.env.NC_REDIS_URL this.activeQueue = process.env.NC_REDIS_URL
? this.jobsQueue ? this.jobsQueue
@ -37,14 +41,42 @@ export class DuplicateController {
@Param('projectId') projectId: string, @Param('projectId') projectId: string,
@Param('baseId') baseId?: string, @Param('baseId') baseId?: string,
) { ) {
const project = await Project.get(projectId);
if (!project) {
throw new Error(`Project not found for id '${projectId}'`);
}
const base = baseId
? await Base.get(baseId)
: (await project.getBases())[0];
if (!base) {
throw new Error(`Base not found!`);
}
const projects = await Project.list({});
const uniqueTitle = generateUniqueName(
`${project.title} copy`,
projects.map((p) => p.title),
);
const dupProject = await this.projectsService.projectCreate({
project: { title: uniqueTitle, status: 'job' },
user: { id: req.user.id },
});
const job = await this.activeQueue.add('duplicate', { const job = await this.activeQueue.add('duplicate', {
projectId, project,
baseId, base,
dupProject,
req: { req: {
user: req.user, user: req.user,
clientIp: req.clientIp, clientIp: req.clientIp,
}, },
}); });
return { id: job.id, name: job.name }; return { id: job.id, name: job.name };
} }
} }

472
packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts
Unescape View File

@ -6,15 +6,12 @@ import {
Process, Process,
Processor, Processor,
} from '@nestjs/bull'; } from '@nestjs/bull';
import { Base, Column, Model, Project } from 'src/models'; import { Column, Model } from 'src/models';
import { Job } from 'bull'; import { Job } from 'bull';
import { ProjectsService } from 'src/services/projects.service'; import { ProjectsService } from 'src/services/projects.service';
import boxen from 'boxen'; import boxen from 'boxen';
import papaparse from 'papaparse'; import papaparse from 'papaparse';
import { import { findWithIdentifier } from 'src/helpers/exportImportHelpers';
findWithIdentifier,
generateUniqueName,
} from 'src/helpers/exportImportHelpers';
import { BulkDataAliasService } from 'src/services/bulk-data-alias.service'; import { BulkDataAliasService } from 'src/services/bulk-data-alias.service';
import { UITypes } from 'nocodb-sdk'; import { UITypes } from 'nocodb-sdk';
import { forwardRef, Inject } from '@nestjs/common'; import { forwardRef, Inject } from '@nestjs/common';
@ -76,295 +73,274 @@ export class DuplicateProcessor {
@Process('duplicate') @Process('duplicate')
async duplicateBase(job: Job) { async duplicateBase(job: Job) {
let start = process.hrtime(); const { project, base, dupProject, req } = job.data;
const debugLog = function (...args: any[]) {
if (DEBUG) {
console.log(...args);
}
};
const elapsedTime = function (label?: string) { try {
const elapsedS = process.hrtime(start)[0].toFixed(3); let start = process.hrtime();
const elapsedMs = process.hrtime(start)[1] / 1000000;
if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
start = process.hrtime();
};
const param: { projectId: string; baseId?: string; req: any } = job.data; const debugLog = function (...args: any[]) {
if (DEBUG) {
console.log(...args);
}
};
const user = (param.req as any).user; const elapsedTime = function (label?: string) {
const elapsedS = process.hrtime(start)[0].toFixed(3);
const elapsedMs = process.hrtime(start)[1] / 1000000;
if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
start = process.hrtime();
};
const project = await Project.get(param.projectId); const user = (req as any).user;
if (!project) {
throw new Error(`Base not found for id '${param.baseId}'`);
}
const base = param?.baseId const models = (await base.getModels()).filter(
? await Base.get(param.baseId) (m) => !m.mm && m.type === 'table',
: (await project.getBases())[0]; );
if (!base) {
throw new Error(`Base not found!`);
}
const models = (await base.getModels()).filter(
(m) => !m.mm && m.type === 'table',
);
const exportedModels = await this.exportService.serializeModels({ const exportedModels = await this.exportService.serializeModels({
modelIds: models.map((m) => m.id), modelIds: models.map((m) => m.id),
}); });
elapsedTime('serializeModels');
if (!exportedModels) {
throw new Error(`Export failed for base '${base.id}'`);
}
const projects = await Project.list({});
const uniqueTitle = generateUniqueName( elapsedTime('serializeModels');
`${project.title} copy`,
projects.map((p) => p.title),
);
const dupProject = await this.projectsService.projectCreate({ if (!exportedModels) {
project: { title: uniqueTitle, status: 'job' }, throw new Error(`Export failed for base '${base.id}'`);
user: { id: user.id }, }
});
this.jobsGateway.jobStatus({
name: job.name,
id: job.id.toString(),
status: 'refresh',
});
const dupBaseId = dupProject.bases[0].id; const dupBaseId = dupProject.bases[0].id;
elapsedTime('projectCreate'); elapsedTime('projectCreate');
const idMap = await this.importService.importModels({ const idMap = await this.importService.importModels({
user, user,
projectId: dupProject.id, projectId: dupProject.id,
baseId: dupBaseId, baseId: dupBaseId,
data: exportedModels, data: exportedModels,
req: param.req, req: req,
}); });
elapsedTime('importModels'); elapsedTime('importModels');
if (!idMap) { if (!idMap) {
throw new Error(`Import failed for base '${base.id}'`); throw new Error(`Import failed for base '${base.id}'`);
} }
const handledLinks = []; const handledLinks = [];
const lChunk: Record<string, any[]> = {}; // colId: { rowId, childId }[] const lChunk: Record<string, any[]> = {}; // colId: { rowId, childId }[]
for (const sourceModel of models) { for (const sourceModel of models) {
const dataStream = new Readable({ const dataStream = new Readable({
read() {}, read() {},
}); });
const linkStream = new Readable({ const linkStream = new Readable({
read() {}, read() {},
}); });
this.exportService.streamModelData({ this.exportService.streamModelData({
dataStream, dataStream,
linkStream, linkStream,
projectId: project.id, projectId: project.id,
modelId: sourceModel.id, modelId: sourceModel.id,
}); });
const headers: string[] = []; const headers: string[] = [];
let chunk = []; let chunk = [];
const model = await Model.get(findWithIdentifier(idMap, sourceModel.id)); const model = await Model.get(
findWithIdentifier(idMap, sourceModel.id),
await new Promise((resolve) => { );
papaparse.parse(dataStream, {
newline: '\r\n', await new Promise((resolve) => {
step: async (results, parser) => { papaparse.parse(dataStream, {
if (!headers.length) { newline: '\r\n',
parser.pause(); step: async (results, parser) => {
for (const header of results.data) { if (!headers.length) {
const id = idMap.get(header); parser.pause();
if (id) { for (const header of results.data) {
const col = await Column.get({ const id = idMap.get(header);
base_id: dupBaseId, if (id) {
colId: id, const col = await Column.get({
});
if (col.colOptions?.type === 'bt') {
const childCol = await Column.get({
base_id: dupBaseId, base_id: dupBaseId,
colId: col.colOptions.fk_child_column_id, colId: id,
}); });
headers.push(childCol.column_name); if (col.colOptions?.type === 'bt') {
const childCol = await Column.get({
base_id: dupBaseId,
colId: col.colOptions.fk_child_column_id,
});
headers.push(childCol.column_name);
} else {
headers.push(col.column_name);
}
} else { } else {
headers.push(col.column_name); debugLog('header not found', header);
} }
} else {
debugLog('header not found', header);
} }
} parser.resume();
parser.resume(); } else {
} else { if (results.errors.length === 0) {
if (results.errors.length === 0) { const row = {};
const row = {}; for (let i = 0; i < headers.length; i++) {
for (let i = 0; i < headers.length; i++) { if (results.data[i] !== '') {
if (results.data[i] !== '') { row[headers[i]] = results.data[i];
row[headers[i]] = results.data[i]; }
} }
} chunk.push(row);
chunk.push(row); if (chunk.length > 1000) {
if (chunk.length > 1000) { parser.pause();
parser.pause(); try {
try { await this.bulkDataService.bulkDataInsert({
await this.bulkDataService.bulkDataInsert({ projectName: dupProject.id,
projectName: dupProject.id, tableName: model.id,
tableName: model.id, body: chunk,
body: chunk, cookie: null,
cookie: null, chunkSize: chunk.length + 1,
chunkSize: chunk.length + 1, foreign_key_checks: false,
foreign_key_checks: false, raw: true,
raw: true, });
}); } catch (e) {
} catch (e) { console.log(e);
console.log(e); }
chunk = [];
parser.resume();
} }
chunk = [];
parser.resume();
} }
} }
} },
}, complete: async () => {
complete: async () => { if (chunk.length > 0) {
if (chunk.length > 0) { try {
try { await this.bulkDataService.bulkDataInsert({
await this.bulkDataService.bulkDataInsert({ projectName: dupProject.id,
projectName: dupProject.id, tableName: model.id,
tableName: model.id, body: chunk,
body: chunk, cookie: null,
cookie: null, chunkSize: chunk.length + 1,
chunkSize: chunk.length + 1, foreign_key_checks: false,
foreign_key_checks: false, raw: true,
raw: true, });
}); } catch (e) {
} catch (e) { console.log(e);
console.log(e); }
chunk = [];
} }
chunk = []; resolve(null);
} },
resolve(null); });
},
}); });
});
const lHeaders: string[] = []; const lHeaders: string[] = [];
const mmParentChild: any = {}; const mmParentChild: any = {};
let pkIndex = -1; let pkIndex = -1;
await new Promise((resolve) => { await new Promise((resolve) => {
papaparse.parse(linkStream, { papaparse.parse(linkStream, {
newline: '\r\n', newline: '\r\n',
step: async (results, parser) => { step: async (results, parser) => {
if (!lHeaders.length) { if (!lHeaders.length) {
parser.pause(); parser.pause();
for (const header of results.data) { for (const header of results.data) {
if (header === 'pk') { if (header === 'pk') {
lHeaders.push(null);
pkIndex = lHeaders.length - 1;
continue;
}
const id = idMap.get(header);
if (id) {
const col = await Column.get({
base_id: dupBaseId,
colId: id,
});
if (
col.uidt === UITypes.LinkToAnotherRecord &&
col.colOptions.fk_mm_model_id &&
handledLinks.includes(col.colOptions.fk_mm_model_id)
) {
lHeaders.push(null); lHeaders.push(null);
} else { pkIndex = lHeaders.length - 1;
continue;
}
const id = idMap.get(header);
if (id) {
const col = await Column.get({
base_id: dupBaseId,
colId: id,
});
if ( if (
col.uidt === UITypes.LinkToAnotherRecord && col.uidt === UITypes.LinkToAnotherRecord &&
col.colOptions.fk_mm_model_id && col.colOptions.fk_mm_model_id &&
!handledLinks.includes(col.colOptions.fk_mm_model_id) handledLinks.includes(col.colOptions.fk_mm_model_id)
) { ) {
const colOptions = lHeaders.push(null);
await col.getColOptions<LinkToAnotherRecordColumn>(); } else {
if (
const vChildCol = await colOptions.getMMChildColumn(); col.uidt === UITypes.LinkToAnotherRecord &&
const vParentCol = await colOptions.getMMParentColumn(); col.colOptions.fk_mm_model_id &&
!handledLinks.includes(col.colOptions.fk_mm_model_id)
mmParentChild[col.colOptions.fk_mm_model_id] = { ) {
parent: vParentCol.column_name, const colOptions =
child: vChildCol.column_name, await col.getColOptions<LinkToAnotherRecordColumn>();
};
const vChildCol = await colOptions.getMMChildColumn();
handledLinks.push(col.colOptions.fk_mm_model_id); const vParentCol = await colOptions.getMMParentColumn();
mmParentChild[col.colOptions.fk_mm_model_id] = {
parent: vParentCol.column_name,
child: vChildCol.column_name,
};
handledLinks.push(col.colOptions.fk_mm_model_id);
}
lHeaders.push(col.colOptions.fk_mm_model_id);
lChunk[col.colOptions.fk_mm_model_id] = [];
} }
lHeaders.push(col.colOptions.fk_mm_model_id);
lChunk[col.colOptions.fk_mm_model_id] = [];
} }
} }
} parser.resume();
parser.resume(); } else {
} else { if (results.errors.length === 0) {
if (results.errors.length === 0) { for (let i = 0; i < lHeaders.length; i++) {
for (let i = 0; i < lHeaders.length; i++) { if (!lHeaders[i]) continue;
if (!lHeaders[i]) continue;
const mm = mmParentChild[lHeaders[i]];
const mm = mmParentChild[lHeaders[i]];
for (const rel of results.data[i].split(',')) {
for (const rel of results.data[i].split(',')) { if (rel.trim() === '') continue;
if (rel.trim() === '') continue; lChunk[lHeaders[i]].push({
lChunk[lHeaders[i]].push({ [mm.parent]: rel,
[mm.parent]: rel, [mm.child]: results.data[pkIndex],
[mm.child]: results.data[pkIndex], });
}); }
} }
} }
} }
} },
}, complete: async () => {
complete: async () => { resolve(null);
resolve(null); },
}, });
}); });
});
elapsedTime(model.title); elapsedTime(model.title);
} }
for (const [k, v] of Object.entries(lChunk)) {
try {
await this.bulkDataService.bulkDataInsert({
projectName: dupProject.id,
tableName: k,
body: v,
cookie: null,
chunkSize: 1000,
foreign_key_checks: false,
raw: true,
});
} catch (e) {
console.log(e);
}
}
for (const [k, v] of Object.entries(lChunk)) { elapsedTime('links');
try { await this.projectsService.projectUpdate({
await this.bulkDataService.bulkDataInsert({ projectId: dupProject.id,
projectName: dupProject.id, project: {
tableName: k, status: null,
body: v, },
cookie: null, });
chunkSize: 1000, } catch (e) {
foreign_key_checks: false, if (dupProject?.id) {
raw: true, await this.projectsService.projectSoftDelete({
projectId: dupProject.id,
}); });
} catch (e) {
console.log(e);
} }
throw e;
} }
elapsedTime('links');
await this.projectsService.projectUpdate({
projectId: dupProject.id,
project: {
status: null,
},
});
} }
} }

Write Preview
Loading…
Cancel
Save