From 29ec7c431a8356c134c16f15752d25ed31731e66 Mon Sep 17 00:00:00 2001 From: mertmit Date: Tue, 18 Apr 2023 12:13:43 +0300 Subject: [PATCH] feat: base export/import Signed-off-by: mertmit --- packages/nocodb/.gitignore | 3 +- .../controllers/exportImport/export.ctl.ts | 19 +- .../controllers/exportImport/import.ctl.ts | 15 +- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 28 +- .../src/lib/services/dbData/bulkData.ts | 4 +- .../lib/services/exportImport/export.svc.ts | 273 ++++++++++++-- .../lib/services/exportImport/import.svc.ts | 333 ++++++++++++++++-- 7 files changed, 604 insertions(+), 71 deletions(-) diff --git a/packages/nocodb/.gitignore b/packages/nocodb/.gitignore index 58c18f71e6..a7d7c847be 100644 --- a/packages/nocodb/.gitignore +++ b/packages/nocodb/.gitignore @@ -19,4 +19,5 @@ noco.db* test_meta.db test_sakila.db test_sakila_*.db -.env \ No newline at end of file +.env +export/** \ No newline at end of file diff --git a/packages/nocodb/src/lib/controllers/exportImport/export.ctl.ts b/packages/nocodb/src/lib/controllers/exportImport/export.ctl.ts index 0d090de7c7..6423ab0ddc 100644 --- a/packages/nocodb/src/lib/controllers/exportImport/export.ctl.ts +++ b/packages/nocodb/src/lib/controllers/exportImport/export.ctl.ts @@ -3,28 +3,17 @@ import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; import { exportService } from '../../services'; import type { Request, Response } from 'express'; -export async function exportBaseSchema(req: Request, res: Response) { +export async function exportBase(req: Request, res: Response) { res.json( - await exportService.exportBaseSchema({ baseId: req.params.baseId }) - ); -} - -export async function exportModelData(req: Request, res: Response) { - res.json( - await exportService.exportModelData({ projectId: req.params.projectId, modelId: req.params.modelId, viewId: req.params.viewId }) + await exportService.exportBase({ baseId: req.params.baseId, path: req.body.path }) ); } const router = Router({ mergeParams: true }); -router.get( +router.post( '/api/v1/db/meta/export/:projectId/:baseId', - ncMetaAclMw(exportBaseSchema, 'exportBaseSchema') -); - -router.get( - '/api/v1/db/meta/export/data/:projectId/:modelId/:viewId?', - ncMetaAclMw(exportModelData, 'exportModelData') + ncMetaAclMw(exportBase, 'exportBase') ); export default router; diff --git a/packages/nocodb/src/lib/controllers/exportImport/import.ctl.ts b/packages/nocodb/src/lib/controllers/exportImport/import.ctl.ts index 80af204134..91b81f0c36 100644 --- a/packages/nocodb/src/lib/controllers/exportImport/import.ctl.ts +++ b/packages/nocodb/src/lib/controllers/exportImport/import.ctl.ts @@ -16,11 +16,24 @@ export async function importModels(req: Request, res: Response) { ); } +export async function importBase(req: Request, res: Response) { + const { body, ...rest } = req; + res.json( + await importService.importBase({ + user: (req as any).user, + projectId: req.params.projectId, + baseId: req.params.baseId, + src: body.src, + req: rest, + }) + ); +} + const router = Router({ mergeParams: true }); router.post( '/api/v1/db/meta/import/:projectId/:baseId', - ncMetaAclMw(importModels, 'importModels') + ncMetaAclMw(importBase, 'importBase') ); export default router; diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index c38e47b8ed..5569bd4bab 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -2066,9 +2066,11 @@ class BaseModelSqlv2 { { chunkSize: _chunkSize = 100, cookie, + foreign_key_checks = true, }: { chunkSize?: number; cookie?: any; + foreign_key_checks?: boolean; } = {} ) { try { @@ -2090,17 +2092,37 @@ class BaseModelSqlv2 { // refer : https://www.sqlite.org/limits.html const chunkSize = this.isSqlite ? 10 : _chunkSize; + const trx = await this.dbDriver.transaction(); + + if (!foreign_key_checks) { + if (this.isPg) { + await trx.raw('set session_replication_role to replica;'); + } else if (this.isMySQL) { + await trx.raw('SET foreign_key_checks = 0;'); + } + } + const response = this.isPg || this.isMssql - ? await this.dbDriver + ? await trx .batchInsert(this.tnPath, insertDatas, chunkSize) .returning(this.model.primaryKey?.column_name) - : await this.dbDriver.batchInsert( + : await trx.batchInsert( this.tnPath, insertDatas, chunkSize ); + if (!foreign_key_checks) { + if (this.isPg) { + await trx.raw('set session_replication_role to origin;'); + } else if (this.isMySQL) { + await trx.raw('SET foreign_key_checks = 1;'); + } + } + + await trx.commit(); + await this.afterBulkInsert(insertDatas, this.dbDriver, cookie); return response; @@ -2719,7 +2741,7 @@ class BaseModelSqlv2 { await this.afterInsert(response, this.dbDriver, cookie); await this.afterAddChild(rowId, childId, cookie); } - + public async afterAddChild(rowId, childId, req): Promise { await Audit.insert({ fk_model_id: this.model.id, diff --git a/packages/nocodb/src/lib/services/dbData/bulkData.ts b/packages/nocodb/src/lib/services/dbData/bulkData.ts index ab83ee0efc..f67edab921 100644 --- a/packages/nocodb/src/lib/services/dbData/bulkData.ts +++ b/packages/nocodb/src/lib/services/dbData/bulkData.ts @@ -38,12 +38,14 @@ export async function bulkDataInsert( param: PathParams & { body: any; cookie: any; + chunkSize?: number; + foreign_key_checks?: boolean; } ) { return await executeBulkOperation({ ...param, operation: 'bulkInsert', - options: [param.body, { cookie: param.cookie }], + options: [param.body, { cookie: param.cookie, foreign_key_checks: param.foreign_key_checks, chunkSize: param.chunkSize }], }); } diff --git a/packages/nocodb/src/lib/services/exportImport/export.svc.ts b/packages/nocodb/src/lib/services/exportImport/export.svc.ts index 9ffdf53985..b176387dff 100644 --- a/packages/nocodb/src/lib/services/exportImport/export.svc.ts +++ b/packages/nocodb/src/lib/services/exportImport/export.svc.ts @@ -1,8 +1,12 @@ import { NcError } from './../../meta/helpers/catchError'; -import { ViewTypes } from 'nocodb-sdk'; -import { Project, Base, Model } from '../../models'; +import { UITypes, ViewTypes } from 'nocodb-sdk'; +import { Project, Base, Model, View, LinkToAnotherRecordColumn } from '../../models'; import { dataService } from '..'; import { getViewAndModelByAliasOrId } from '../dbData/helpers'; +import { Readable } from 'stream'; +import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; +import { unparse } from 'papaparse'; +import { IStorageAdapterV2 } from 'nc-plugin'; /* { @@ -24,6 +28,22 @@ import { getViewAndModelByAliasOrId } from '../dbData/helpers'; } */ +async function generateBaseIdMap(base: Base, idMap: Map) { + idMap.set(base.project_id, base.project_id); + idMap.set(base.id, `${base.project_id}::${base.id}`); + const models = await base.getModels(); + + for (const md of models) { + idMap.set(md.id, `${base.project_id}::${base.id}::${md.id}`); + await md.getColumns(); + for (const column of md.columns) { + idMap.set(column.id, `${idMap.get(md.id)}::${column.id}`); + } + } + + return models; +} + async function serializeModels(param: { modelId: string[] }) { const serializedModels = []; @@ -49,31 +69,13 @@ async function serializeModels(param: { modelId: string[] }) { if (!fndBase) bases.push(base); if (!modelsMap.has(base.id)) { - const all_models = await base.getModels(); - - for (const md of all_models) { - idMap.set(md.id, `${project.id}::${base.id}::${md.id}`); - await md.getColumns(); - for (const column of md.columns) { - idMap.set(column.id, `${idMap.get(md.id)}::${column.id}`); - } - } - - modelsMap.set(base.id, all_models); + modelsMap.set(base.id, await generateBaseIdMap(base, idMap)); } - idMap.set(project.id, project.id); - idMap.set(base.id, `${project.id}::${base.id}`); - idMap.set(model.id, `${idMap.get(base.id)}::${model.id}`); - await model.getColumns(); await model.getViews(); for (const column of model.columns) { - idMap.set( - column.id, - `${idMap.get(model.id)}::${column.id}` - ); await column.getColOptions(); if (column.colOptions) { for (const [k, v] of Object.entries(column.colOptions)) { @@ -248,6 +250,181 @@ async function serializeModels(param: { modelId: string[] }) { return serializedModels; } +async function exportModelData(param: { + storageAdapter: IStorageAdapterV2; + path: string; + projectId: string; + modelId: string; + viewId?: string; +}) { + const { model, view } = await getViewAndModelByAliasOrId({ + projectName: param.projectId, + tableName: param.modelId, + viewName: param.viewId, + }); + + await model.getColumns(); + + const hasLink = model.columns.some((c) => c.uidt === UITypes.LinkToAnotherRecord && c.colOptions?.type === 'mm'); + + const pkMap = new Map(); + + for (const column of model.columns.filter((c) => c.uidt === UITypes.LinkToAnotherRecord && c.colOptions?.type !== 'hm')) { + const relatedTable = await ( + (await column.getColOptions()) as LinkToAnotherRecordColumn + ).getRelatedTable(); + + await relatedTable.getColumns(); + + pkMap.set(column.id, relatedTable.primaryKey.title); + } + + const readableStream = new Readable({ + read() {}, + }); + + const readableLinkStream = new Readable({ + read() {}, + }); + + readableStream.setEncoding('utf8'); + + readableLinkStream.setEncoding('utf8'); + + const storageAdapter = param.storageAdapter; + + const uploadPromise = storageAdapter.fileCreateByStream( + `${param.path}/${model.id}.csv`, + readableStream + ); + + const uploadLinkPromise = hasLink + ? storageAdapter.fileCreateByStream( + `${param.path}/${model.id}_links.csv`, + readableLinkStream + ) + : Promise.resolve(); + + const limit = 100; + let offset = 0; + + const primaryKey = model.columns.find((c) => c.pk); + + const formatData = (data: any) => { + const linkData = []; + for (const row of data) { + const pkValue = primaryKey ? row[primaryKey.title] : undefined; + const linkRow = {}; + for (const [k, v] of Object.entries(row)) { + const col = model.columns.find((c) => c.title === k); + if (col) { + if (col.pk) linkRow['pk'] = pkValue; + const colId = `${col.project_id}::${col.base_id}::${col.fk_model_id}::${col.id}`; + switch(col.uidt) { + case UITypes.LinkToAnotherRecord: + if (col.system || col.colOptions.type === 'hm') break; + const pkList = []; + + const links = Array.isArray(v) ? v : [v]; + + for (const link of links) { + if (link) { + for (const [k, val] of Object.entries(link)) { + if (k === pkMap.get(col.id)) { + pkList.push(val); + } + } + } + } + + if (col.colOptions.type === 'mm') { + linkRow[colId] = pkList.join(','); + } else { + row[colId] = pkList[0]; + } + break; + case UITypes.Attachment: + try { + row[colId] = JSON.stringify(v); + } catch (e) { + row[colId] = v; + } + break; + case UITypes.ForeignKey: + case UITypes.Formula: + case UITypes.Lookup: + case UITypes.Rollup: + case UITypes.Rating: + case UITypes.Barcode: + // skip these types + break; + default: + row[colId] = v; + break; + } + delete row[k]; + } + } + linkData.push(linkRow); + } + return { data, linkData }; + } + + try { + await recursiveRead(formatData, readableStream, readableLinkStream, model, view, offset, limit, true); + await uploadPromise; + await uploadLinkPromise; + } catch (e) { + await storageAdapter.fileDelete(`${param.path}/${model.id}.csv`); + await storageAdapter.fileDelete(`${param.path}/${model.id}_links.csv`); + console.error(e); + throw e; + } + + return true; +} + +async function recursiveRead( + formatter: Function, + stream: Readable, + linkStream: Readable, + model: Model, + view: View, + offset: number, + limit: number, + header = false +): Promise { + return new Promise((resolve, reject) => { + dataService + .getDataList({ model, view, query: { limit, offset } }) + .then((result) => { + try { + if (!header) { + stream.push('\r\n'); + linkStream.push('\r\n'); + } + const { data, linkData } = formatter(result.list); + stream.push(unparse(data, { header })); + linkStream.push(unparse(linkData, { header })); + if (result.pageInfo.isLastPage) { + stream.push(null); + linkStream.push(null); + resolve(); + } else { + recursiveRead(formatter, stream, linkStream, model, view, offset + limit, limit).then(resolve); + } + } catch (e) { + reject(e); + } + }); + }); +} + +function clearPrefix(text: string, prefix?: string) { + if (!prefix || prefix.length === 0) return text; + return text.replace(new RegExp(`^${prefix}_?`), ''); +} + export async function exportBaseSchema(param: { baseId: string }) { const base = await Base.get(param.baseId); @@ -255,7 +432,7 @@ export async function exportBaseSchema(param: { baseId: string }) { const project = await Project.get(base.project_id); - const models = (await base.getModels()).filter((m) => !m.mm); + const models = (await base.getModels()).filter((m) => !m.mm && m.type === 'table'); const exportedModels = await serializeModels({ modelId: models.map(m => m.id) }); @@ -264,7 +441,53 @@ export async function exportBaseSchema(param: { baseId: string }) { return exportData; } -const clearPrefix = (text: string, prefix?: string) => { - if (!prefix || prefix.length === 0) return text; - return text.replace(new RegExp(`^${prefix}_?`), ''); +export async function exportBase(param: { path: string; baseId: string }) { + const base = await Base.get(param.baseId); + + if (!base) return NcError.badRequest(`Base not found for id '${param.baseId}'`); + + const project = await Project.get(base.project_id); + + const models = (await base.getModels()).filter((m) => !m.mm && m.type === 'table'); + + const exportedModels = await serializeModels({ modelId: models.map(m => m.id) }); + + const exportData = { id: `${project.id}::${base.id}`, entity: 'base', models: exportedModels }; + + const storageAdapter = await NcPluginMgrv2.storageAdapter(); + + const destPath = `export/${project.id}/${base.id}/${param.path}/schema.json`; + + try { + + const readableStream = new Readable({ + read() {}, + }); + + readableStream.setEncoding('utf8'); + + readableStream.push(JSON.stringify(exportData)); + + readableStream.push(null); + + await storageAdapter.fileCreateByStream( + destPath, + readableStream + ); + + for (const model of models) { + await exportModelData({ + storageAdapter, + path: `export/${project.id}/${base.id}/${param.path}/data`, + projectId: project.id, + modelId: model.id, + }); + } + + } catch (e) { + console.error(e); + return NcError.internalServerError('Error while exporting base'); + } + + return true; } diff --git a/packages/nocodb/src/lib/services/exportImport/import.svc.ts b/packages/nocodb/src/lib/services/exportImport/import.svc.ts index adee428c6e..31fd4b32e6 100644 --- a/packages/nocodb/src/lib/services/exportImport/import.svc.ts +++ b/packages/nocodb/src/lib/services/exportImport/import.svc.ts @@ -1,14 +1,16 @@ import type { ViewCreateReqType } from 'nocodb-sdk'; import { UITypes, ViewTypes } from 'nocodb-sdk'; -import { tableService, gridViewService, filterService, viewColumnService, gridViewColumnService, sortService, formViewService, galleryViewService, kanbanViewService, formViewColumnService, columnService } from '..'; +import { tableService, gridViewService, filterService, viewColumnService, gridViewColumnService, sortService, formViewService, galleryViewService, kanbanViewService, formViewColumnService, columnService, bulkDataService } from '..'; import { NcError } from '../../meta/helpers/catchError'; -import { Project, Base, User, View, Model } from '../../models'; +import { Project, Base, User, View, Model, Column, LinkToAnotherRecordColumn } from '../../models'; +import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; +import papaparse from 'papaparse'; export async function importModels(param: { user: User; projectId: string; baseId: string; - data: { model: any; views: any[] }[]; + data: { models: { model: any; views: any[] }[] } | { model: any; views: any[] }[]; req: any; }) { @@ -25,6 +27,8 @@ export async function importModels(param: { const tableReferences = new Map(); const linkMap = new Map(); + + param.data = Array.isArray(param.data) ? param.data : param.data.models; // create tables with static columns for (const data of param.data) { @@ -83,7 +87,7 @@ export async function importModels(param: { if (!linkMap.has(colOptions.fk_mm_model_id)) { // delete col.column_name as it is not required and will cause ajv error (null for LTAR) delete col.column_name; - + const freshModelData = await columnService.columnAdd({ tableId: table.id, column: withoutId({ @@ -118,14 +122,16 @@ export async function importModels(param: { if (nColumn?.colOptions?.fk_mm_model_id === linkMap.get(colOptions.fk_mm_model_id) && nColumn.id !== idMap.get(col.id)) { idMap.set(childColumn.id, nColumn.id); - await columnService.columnUpdate({ - columnId: nColumn.id, - column: { - ...nColumn, - column_name: childColumn.title, - title: childColumn.title, - }, - }); + if (nColumn.title !== childColumn.title) { + await columnService.columnUpdate({ + columnId: nColumn.id, + column: { + ...nColumn, + column_name: childColumn.title, + title: childColumn.title, + }, + }); + } break; } } @@ -153,7 +159,8 @@ export async function importModels(param: { for (const nColumn of freshModelData.columns) { if (nColumn.title === col.title) { idMap.set(col.id, nColumn.id); - linkMap.set(colOptions.fk_index_name, nColumn.colOptions.fk_index_name); + idMap.set(colOptions.fk_parent_column_id, nColumn.colOptions.fk_parent_column_id); + idMap.set(colOptions.fk_child_column_id, nColumn.colOptions.fk_child_column_id); break; } } @@ -162,20 +169,31 @@ export async function importModels(param: { if (colOptions.fk_related_model_id !== modelData.id) await childModel.getColumns(); - const childColumn = param.data.find(a => a.model.id === colOptions.fk_related_model_id).model.columns.find(a => a.colOptions?.fk_index_name === colOptions.fk_index_name && a.id !== col.id); + const childColumn = param.data + .find((a) => a.model.id === colOptions.fk_related_model_id) + .model.columns.find( + (a) => + a.colOptions?.fk_parent_column_id === + colOptions.fk_parent_column_id && + a.colOptions?.fk_child_column_id === + colOptions.fk_child_column_id && + a.id !== col.id + ); for (const nColumn of childModel.columns) { - if (nColumn?.colOptions?.fk_index_name === linkMap.get(colOptions.fk_index_name) && nColumn.id !== idMap.get(col.id)) { + if (nColumn.id !== idMap.get(col.id) && nColumn.colOptions?.fk_parent_column_id === idMap.get(colOptions.fk_parent_column_id) && nColumn.colOptions?.fk_child_column_id === idMap.get(colOptions.fk_child_column_id)) { idMap.set(childColumn.id, nColumn.id); - await columnService.columnUpdate({ - columnId: nColumn.id, - column: { - ...nColumn, - column_name: childColumn.title, - title: childColumn.title, - }, - }); + if (nColumn.title !== childColumn.title) { + await columnService.columnUpdate({ + columnId: nColumn.id, + column: { + ...nColumn, + column_name: childColumn.title, + title: childColumn.title, + }, + }); + } break; } } @@ -385,6 +403,8 @@ export async function importModels(param: { } } } + + return idMap; } async function createView(idMap: Map, md: Model, vw: Partial, views: View[]): Promise { @@ -529,9 +549,272 @@ function getParentIdentifier(id: string) { arr.pop(); return arr.join('::'); } -/* + function getEntityIdentifier(id: string) { const arr = id.split('::'); return arr.pop(); } -*/ + +function findWithIdentifier(map: Map, id: string) { + for (const key of map.keys()) { + if (getEntityIdentifier(key) === id) { + return map.get(key); + } + } + return undefined; +} + +export async function importBase(param: { + user: User; + projectId: string; + baseId: string; + src: { type: 'local' | 'url' | 'file'; path?: string; url?: string; file?: any }; + req: any; +}) { + const { user, projectId, baseId, src, req } = param; + + let start = process.hrtime(); + + let elapsed_time = function(label: string){ + const elapsedS = (process.hrtime(start)[0]).toFixed(3); + const elapsedMs = process.hrtime(start)[1] / 1000000; + console.log(`${label}: ${elapsedS}s ${elapsedMs}ms`); + start = process.hrtime(); + } + + switch (src.type) { + case 'local': + const path = src.path.replace(/\/$/, ''); + + const storageAdapter = await NcPluginMgrv2.storageAdapter(); + + try { + const schema = JSON.parse(await storageAdapter.fileRead(`${path}/schema.json`)); + + elapsed_time('read schema'); + + // store fk_mm_model_id (mm) to link once + const handledLinks = []; + + const idMap = await importModels({ + user, + projectId, + baseId, + data: schema, + req, + }); + + elapsed_time('import models'); + + if (idMap) { + const files = await storageAdapter.getDirectoryList(`${path}/data`); + const dataFiles = files.filter((file) => !file.match(/_links\.csv$/)); + const linkFiles = files.filter((file) => file.match(/_links\.csv$/)); + + for (const file of dataFiles) { + const readStream = await storageAdapter.fileReadByStream( + `${path}/data/${file}` + ); + + const headers: string[] = []; + let chunk = []; + + const modelId = findWithIdentifier( + idMap, + file.replace(/\.csv$/, '') + ); + + const model = await Model.get(modelId); + + console.log(`Importing ${model.title}...`); + + await new Promise(async (resolve) => { + papaparse.parse(readStream, { + newline: '\r\n', + step: async function (results, parser) { + if (!headers.length) { + parser.pause(); + for (const header of results.data) { + const id = idMap.get(header); + if (id) { + const col = await Column.get({ + base_id: baseId, + colId: id, + }); + if (col.colOptions?.type === 'bt') { + const childCol = await Column.get({ + base_id: baseId, + colId: col.colOptions.fk_child_column_id, + }); + headers.push(childCol.title); + } else { + headers.push(col.title); + } + + } else { + console.log(header); + } + } + parser.resume(); + } else { + if (results.errors.length === 0) { + const row = {}; + for (let i = 0; i < headers.length; i++) { + if (results.data[i] !== '') { + row[headers[i]] = results.data[i]; + } + } + chunk.push(row); + if (chunk.length > 1000) { + parser.pause(); + elapsed_time('before chunk'); + await bulkDataService.bulkDataInsert({ + projectName: projectId, + tableName: modelId, + body: chunk, + cookie: null, + chunkSize: 1000, + foreign_key_checks: false + }); + chunk = []; + elapsed_time('after chunk'); + parser.resume(); + } + } + } + }, + complete: async function () { + if (chunk.length > 0) { + elapsed_time('before chunk'); + await bulkDataService.bulkDataInsert({ + projectName: projectId, + tableName: modelId, + body: chunk, + cookie: null, + foreign_key_checks: false + }); + chunk = []; + elapsed_time('after chunk'); + } + resolve(null); + }, + }); + }); + } + + for (const file of linkFiles) { + const readStream = await storageAdapter.fileReadByStream( + `${path}/data/${file}` + ); + + const headers: string[] = []; + const mmParentChild: any = {}; + let chunk: Record = {}; // colId: { rowId, childId }[] + + const modelId = findWithIdentifier( + idMap, + file.replace(/_links\.csv$/, '') + ); + const model = await Model.get(modelId); + + let pkIndex = -1; + + console.log(`Linking ${model.title}...`); + + await new Promise(async (resolve) => { + papaparse.parse(readStream, { + newline: '\r\n', + step: async function (results, parser) { + if (!headers.length) { + parser.pause(); + for (const header of results.data) { + if (header === 'pk') { + headers.push(null); + pkIndex = headers.length - 1; + continue; + } + const id = idMap.get(header); + if (id) { + const col = await Column.get({ + base_id: baseId, + colId: id, + }); + if ( + col.uidt === UITypes.LinkToAnotherRecord && + col.colOptions.fk_mm_model_id && + handledLinks.includes(col.colOptions.fk_mm_model_id) + ) { + headers.push(null); + } else { + if ( + col.uidt === UITypes.LinkToAnotherRecord && + col.colOptions.fk_mm_model_id && + !handledLinks.includes( + col.colOptions.fk_mm_model_id + ) + ) { + const colOptions = await col.getColOptions(); + + const vChildCol = await colOptions.getMMChildColumn(); + const vParentCol = await colOptions.getMMParentColumn(); + + mmParentChild[col.colOptions.fk_mm_model_id] = { + parent: vParentCol.title, + child: vChildCol.title, + } + + handledLinks.push(col.colOptions.fk_mm_model_id); + } + headers.push(col.colOptions.fk_mm_model_id); + chunk[col.colOptions.fk_mm_model_id] = [] + } + } + } + parser.resume(); + } else { + if (results.errors.length === 0) { + for (let i = 0; i < headers.length; i++) { + if (!headers[i]) continue; + + const mm = mmParentChild[headers[i]]; + + for (const rel of results.data[i].split(',')) { + if (rel.trim() === '') continue; + chunk[headers[i]].push({ [mm.parent]: rel, [mm.child]: results.data[pkIndex] }); + } + } + } + } + }, + complete: async function () { + for (const [k, v] of Object.entries(chunk)) { + try { + await bulkDataService.bulkDataInsert({ + projectName: projectId, + tableName: k, + body: v, + cookie: null, + chunkSize: 1000, + foreign_key_checks: false + }); + } catch (e) { + console.log('linkError'); + console.log(e); + } + } + resolve(null); + }, + }); + }); + } + } + } catch (e) { + throw new Error(e); + } + break; + case 'url': + break; + case 'file': + break; + } +}