github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: improved duplicate logic 

Signed-off-by: mertmit <mertmit99@gmail.com>
feat/export-nest-dir-restructure
mertmit 2 years ago committed by starbirdtech383
parent
849581d1f5
commit
09e9e525f6
3 changed files with 286 additions and 260 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 38
      packages/nc-gui/pages/index/index/index.vue
  2. 36
      packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts
  3. 472
      packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts

38
packages/nc-gui/pages/index/index/index.vue
Unescape View File

@ -39,9 +39,17 @@ const filterQuery = ref('')
const projects = ref<ProjectType[]>()
const activePage = ref(1)
const pageChange = (p: number) => {
activePage.value = p
}
const loadProjects = async () => {
const lastPage = activePage.value
const response = await api.project.list({})
projects.value = response.list
activePage.value = lastPage
}
const filteredProjects = computed(
@ -85,12 +93,14 @@ const duplicateProject = (project: ProjectType) => {
try {
const jobData = await api.project.duplicate(project.id as string)
await loadProjects()
$events.subscribe(jobData.name, jobData.id, async (data: { status: string }) => {
console.log('dataCB', data)
if (data.status === 'completed' || data.status === 'refresh') {
if (data.status === 'completed') {
await loadProjects()
} else if (data.status === 'failed') {
message.error('Failed to duplicate project')
await loadProjects()
}
})
@ -224,7 +234,7 @@ const copyProjectMeta = async () => {
v-else
:custom-row="customRow"
:data-source="filteredProjects"
:pagination="{ position: ['bottomCenter'] }"
:pagination="{ 'position': ['bottomCenter'], 'current': activePage, 'onUpdate:current': pageChange }"
:table-layout="md ? 'auto' : 'fixed'"
>
<template #emptyText>
@ -301,19 +311,27 @@ const copyProjectMeta = async () => {
@click.stop="navigateTo(`/${text}`)"
/>
<component
:is="iconMap.copy"
v-e="['c:project:duplicate']"
class="nc-action-btn"
@click.stop="duplicateProject(record)"
/>
<component
:is="iconMap.delete"
class="nc-action-btn"
:data-testid="`delete-project-${record.title}`"
@click.stop="deleteProject(record)"
/>
<a-dropdown :trigger="['click']" overlay-class-name="nc-dropdown-import-menu" @click.stop>
<GeneralIcon icon="threeDotVertical" class="nc-import-menu outline-0" />
<template #overlay>
<a-menu class="!py-0 rounded text-sm">
<a-menu-item key="duplicate" v-e="['c:project:duplicate']" @click.stop="duplicateProject(record)">
<div class="color-transition nc-project-menu-item group">
<GeneralIcon icon="copy" class="group-hover:text-accent" />
Duplicate
</div>
</a-menu-item>
</a-menu>
</template>
</a-dropdown>
</div>
</template>
</a-table-column>

36
packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts
Unescape View File

@ -14,6 +14,9 @@ import {
Acl,
ExtractProjectIdMiddleware,
} from 'src/middlewares/extract-project-id/extract-project-id.middleware';
import { ProjectsService } from 'src/services/projects.service';
import { Base, Project } from 'src/models';
import { generateUniqueName } from 'src/helpers/exportImportHelpers';
import { QueueService } from '../fallback-queue.service';
@Controller()
@ -23,6 +26,7 @@ export class DuplicateController {
constructor(
@InjectQueue('jobs') private readonly jobsQueue: Queue,
private readonly fallbackQueueService: QueueService,
private readonly projectsService: ProjectsService,
) {
this.activeQueue = process.env.NC_REDIS_URL
? this.jobsQueue
@ -37,14 +41,42 @@ export class DuplicateController {
@Param('projectId') projectId: string,
@Param('baseId') baseId?: string,
) {
const project = await Project.get(projectId);
if (!project) {
throw new Error(`Project not found for id '${projectId}'`);
}
const base = baseId
? await Base.get(baseId)
: (await project.getBases())[0];
if (!base) {
throw new Error(`Base not found!`);
}
const projects = await Project.list({});
const uniqueTitle = generateUniqueName(
`${project.title} copy`,
projects.map((p) => p.title),
);
const dupProject = await this.projectsService.projectCreate({
project: { title: uniqueTitle, status: 'job' },
user: { id: req.user.id },
});
const job = await this.activeQueue.add('duplicate', {
projectId,
baseId,
project,
base,
dupProject,
req: {
user: req.user,
clientIp: req.clientIp,
},
});
return { id: job.id, name: job.name };
}
}

472
packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts
Unescape View File

@ -6,15 +6,12 @@ import {
Process,
Processor,
} from '@nestjs/bull';
import { Base, Column, Model, Project } from 'src/models';
import { Column, Model } from 'src/models';
import { Job } from 'bull';
import { ProjectsService } from 'src/services/projects.service';
import boxen from 'boxen';
import papaparse from 'papaparse';
import {
findWithIdentifier,
generateUniqueName,
} from 'src/helpers/exportImportHelpers';
import { findWithIdentifier } from 'src/helpers/exportImportHelpers';
import { BulkDataAliasService } from 'src/services/bulk-data-alias.service';
import { UITypes } from 'nocodb-sdk';
import { forwardRef, Inject } from '@nestjs/common';
@ -76,295 +73,274 @@ export class DuplicateProcessor {
@Process('duplicate')
async duplicateBase(job: Job) {
let start = process.hrtime();
const debugLog = function (...args: any[]) {
if (DEBUG) {
console.log(...args);
}
};
const { project, base, dupProject, req } = job.data;
const elapsedTime = function (label?: string) {
const elapsedS = process.hrtime(start)[0].toFixed(3);
const elapsedMs = process.hrtime(start)[1] / 1000000;
if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
start = process.hrtime();
};
try {
let start = process.hrtime();
const param: { projectId: string; baseId?: string; req: any } = job.data;
const debugLog = function (...args: any[]) {
if (DEBUG) {
console.log(...args);
}
};
const user = (param.req as any).user;
const elapsedTime = function (label?: string) {
const elapsedS = process.hrtime(start)[0].toFixed(3);
const elapsedMs = process.hrtime(start)[1] / 1000000;
if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
start = process.hrtime();
};
const project = await Project.get(param.projectId);
if (!project) {
throw new Error(`Base not found for id '${param.baseId}'`);
}
const user = (req as any).user;
const base = param?.baseId
? await Base.get(param.baseId)
: (await project.getBases())[0];
if (!base) {
throw new Error(`Base not found!`);
}
const models = (await base.getModels()).filter(
(m) => !m.mm && m.type === 'table',
);
const models = (await base.getModels()).filter(
(m) => !m.mm && m.type === 'table',
);
const exportedModels = await this.exportService.serializeModels({
modelIds: models.map((m) => m.id),
});
elapsedTime('serializeModels');
if (!exportedModels) {
throw new Error(`Export failed for base '${base.id}'`);
}
const projects = await Project.list({});
const exportedModels = await this.exportService.serializeModels({
modelIds: models.map((m) => m.id),
});
const uniqueTitle = generateUniqueName(
`${project.title} copy`,
projects.map((p) => p.title),
);
elapsedTime('serializeModels');
const dupProject = await this.projectsService.projectCreate({
project: { title: uniqueTitle, status: 'job' },
user: { id: user.id },
});
this.jobsGateway.jobStatus({
name: job.name,
id: job.id.toString(),
status: 'refresh',
});
if (!exportedModels) {
throw new Error(`Export failed for base '${base.id}'`);
}
const dupBaseId = dupProject.bases[0].id;
const dupBaseId = dupProject.bases[0].id;
elapsedTime('projectCreate');
elapsedTime('projectCreate');
const idMap = await this.importService.importModels({
user,
projectId: dupProject.id,
baseId: dupBaseId,
data: exportedModels,
req: param.req,
});
const idMap = await this.importService.importModels({
user,
projectId: dupProject.id,
baseId: dupBaseId,
data: exportedModels,
req: req,
});
elapsedTime('importModels');
elapsedTime('importModels');
if (!idMap) {
throw new Error(`Import failed for base '${base.id}'`);
}
if (!idMap) {
throw new Error(`Import failed for base '${base.id}'`);
}
const handledLinks = [];
const lChunk: Record<string, any[]> = {}; // colId: { rowId, childId }[]
const handledLinks = [];
const lChunk: Record<string, any[]> = {}; // colId: { rowId, childId }[]
for (const sourceModel of models) {
const dataStream = new Readable({
read() {},
});
for (const sourceModel of models) {
const dataStream = new Readable({
read() {},
});
const linkStream = new Readable({
read() {},
});
const linkStream = new Readable({
read() {},
});
this.exportService.streamModelData({
dataStream,
linkStream,
projectId: project.id,
modelId: sourceModel.id,
});
this.exportService.streamModelData({
dataStream,
linkStream,
projectId: project.id,
modelId: sourceModel.id,
});
const headers: string[] = [];
let chunk = [];
const model = await Model.get(findWithIdentifier(idMap, sourceModel.id));
await new Promise((resolve) => {
papaparse.parse(dataStream, {
newline: '\r\n',
step: async (results, parser) => {
if (!headers.length) {
parser.pause();
for (const header of results.data) {
const id = idMap.get(header);
if (id) {
const col = await Column.get({
base_id: dupBaseId,
colId: id,
});
if (col.colOptions?.type === 'bt') {
const childCol = await Column.get({
const headers: string[] = [];
let chunk = [];
const model = await Model.get(
findWithIdentifier(idMap, sourceModel.id),
);
await new Promise((resolve) => {
papaparse.parse(dataStream, {
newline: '\r\n',
step: async (results, parser) => {
if (!headers.length) {
parser.pause();
for (const header of results.data) {
const id = idMap.get(header);
if (id) {
const col = await Column.get({
base_id: dupBaseId,
colId: col.colOptions.fk_child_column_id,
colId: id,
});
headers.push(childCol.column_name);
if (col.colOptions?.type === 'bt') {
const childCol = await Column.get({
base_id: dupBaseId,
colId: col.colOptions.fk_child_column_id,
});
headers.push(childCol.column_name);
} else {
headers.push(col.column_name);
}
} else {
headers.push(col.column_name);
debugLog('header not found', header);
}
} else {
debugLog('header not found', header);
}
}
parser.resume();
} else {
if (results.errors.length === 0) {
const row = {};
for (let i = 0; i < headers.length; i++) {
if (results.data[i] !== '') {
row[headers[i]] = results.data[i];
parser.resume();
} else {
if (results.errors.length === 0) {
const row = {};
for (let i = 0; i < headers.length; i++) {
if (results.data[i] !== '') {
row[headers[i]] = results.data[i];
}
}
}
chunk.push(row);
if (chunk.length > 1000) {
parser.pause();
try {
await this.bulkDataService.bulkDataInsert({
projectName: dupProject.id,
tableName: model.id,
body: chunk,
cookie: null,
chunkSize: chunk.length + 1,
foreign_key_checks: false,
raw: true,
});
} catch (e) {
console.log(e);
chunk.push(row);
if (chunk.length > 1000) {
parser.pause();
try {
await this.bulkDataService.bulkDataInsert({
projectName: dupProject.id,
tableName: model.id,
body: chunk,
cookie: null,
chunkSize: chunk.length + 1,
foreign_key_checks: false,
raw: true,
});
} catch (e) {
console.log(e);
}
chunk = [];
parser.resume();
}
chunk = [];
parser.resume();
}
}
}
},
complete: async () => {
if (chunk.length > 0) {
try {
await this.bulkDataService.bulkDataInsert({
projectName: dupProject.id,
tableName: model.id,
body: chunk,
cookie: null,
chunkSize: chunk.length + 1,
foreign_key_checks: false,
raw: true,
});
} catch (e) {
console.log(e);
},
complete: async () => {
if (chunk.length > 0) {
try {
await this.bulkDataService.bulkDataInsert({
projectName: dupProject.id,
tableName: model.id,
body: chunk,
cookie: null,
chunkSize: chunk.length + 1,
foreign_key_checks: false,
raw: true,
});
} catch (e) {
console.log(e);
}
chunk = [];
}
chunk = [];
}
resolve(null);
},
resolve(null);
},
});
});
});
const lHeaders: string[] = [];
const mmParentChild: any = {};
let pkIndex = -1;
await new Promise((resolve) => {
papaparse.parse(linkStream, {
newline: '\r\n',
step: async (results, parser) => {
if (!lHeaders.length) {
parser.pause();
for (const header of results.data) {
if (header === 'pk') {
lHeaders.push(null);
pkIndex = lHeaders.length - 1;
continue;
}
const id = idMap.get(header);
if (id) {
const col = await Column.get({
base_id: dupBaseId,
colId: id,
});
if (
col.uidt === UITypes.LinkToAnotherRecord &&
col.colOptions.fk_mm_model_id &&
handledLinks.includes(col.colOptions.fk_mm_model_id)
) {
const lHeaders: string[] = [];
const mmParentChild: any = {};
let pkIndex = -1;
await new Promise((resolve) => {
papaparse.parse(linkStream, {
newline: '\r\n',
step: async (results, parser) => {
if (!lHeaders.length) {
parser.pause();
for (const header of results.data) {
if (header === 'pk') {
lHeaders.push(null);
} else {
pkIndex = lHeaders.length - 1;
continue;
}
const id = idMap.get(header);
if (id) {
const col = await Column.get({
base_id: dupBaseId,
colId: id,
});
if (
col.uidt === UITypes.LinkToAnotherRecord &&
col.colOptions.fk_mm_model_id &&
!handledLinks.includes(col.colOptions.fk_mm_model_id)
handledLinks.includes(col.colOptions.fk_mm_model_id)
) {
const colOptions =
await col.getColOptions<LinkToAnotherRecordColumn>();
const vChildCol = await colOptions.getMMChildColumn();
const vParentCol = await colOptions.getMMParentColumn();
mmParentChild[col.colOptions.fk_mm_model_id] = {
parent: vParentCol.column_name,
child: vChildCol.column_name,
};
handledLinks.push(col.colOptions.fk_mm_model_id);
lHeaders.push(null);
} else {
if (
col.uidt === UITypes.LinkToAnotherRecord &&
col.colOptions.fk_mm_model_id &&
!handledLinks.includes(col.colOptions.fk_mm_model_id)
) {
const colOptions =
await col.getColOptions<LinkToAnotherRecordColumn>();
const vChildCol = await colOptions.getMMChildColumn();
const vParentCol = await colOptions.getMMParentColumn();
mmParentChild[col.colOptions.fk_mm_model_id] = {
parent: vParentCol.column_name,
child: vChildCol.column_name,
};
handledLinks.push(col.colOptions.fk_mm_model_id);
}
lHeaders.push(col.colOptions.fk_mm_model_id);
lChunk[col.colOptions.fk_mm_model_id] = [];
}
lHeaders.push(col.colOptions.fk_mm_model_id);
lChunk[col.colOptions.fk_mm_model_id] = [];
}
}
}
parser.resume();
} else {
if (results.errors.length === 0) {
for (let i = 0; i < lHeaders.length; i++) {
if (!lHeaders[i]) continue;
const mm = mmParentChild[lHeaders[i]];
for (const rel of results.data[i].split(',')) {
if (rel.trim() === '') continue;
lChunk[lHeaders[i]].push({
[mm.parent]: rel,
[mm.child]: results.data[pkIndex],
});
parser.resume();
} else {
if (results.errors.length === 0) {
for (let i = 0; i < lHeaders.length; i++) {
if (!lHeaders[i]) continue;
const mm = mmParentChild[lHeaders[i]];
for (const rel of results.data[i].split(',')) {
if (rel.trim() === '') continue;
lChunk[lHeaders[i]].push({
[mm.parent]: rel,
[mm.child]: results.data[pkIndex],
});
}
}
}
}
}
},
complete: async () => {
resolve(null);
},
},
complete: async () => {
resolve(null);
},
});
});
});
elapsedTime(model.title);
}
elapsedTime(model.title);
}
for (const [k, v] of Object.entries(lChunk)) {
try {
await this.bulkDataService.bulkDataInsert({
projectName: dupProject.id,
tableName: k,
body: v,
cookie: null,
chunkSize: 1000,
foreign_key_checks: false,
raw: true,
});
} catch (e) {
console.log(e);
}
}
for (const [k, v] of Object.entries(lChunk)) {
try {
await this.bulkDataService.bulkDataInsert({
projectName: dupProject.id,
tableName: k,
body: v,
cookie: null,
chunkSize: 1000,
foreign_key_checks: false,
raw: true,
elapsedTime('links');
await this.projectsService.projectUpdate({
projectId: dupProject.id,
project: {
status: null,
},
});
} catch (e) {
if (dupProject?.id) {
await this.projectsService.projectSoftDelete({
projectId: dupProject.id,
});
} catch (e) {
console.log(e);
}
throw e;
}
elapsedTime('links');
await this.projectsService.projectUpdate({
projectId: dupProject.id,
project: {
status: null,
},
});
}
}

Write Preview
Loading…
Cancel
Save