feat: base export/import

Signed-off-by: mertmit <mertmit99@gmail.com>

packages/nocodb/.gitignore

@@ -19,4 +19,5 @@ noco.db*
 test_meta.db
 test_sakila.db
 test_sakila_*.db
-.env
+.env
+export/**

packages/nocodb/src/lib/controllers/exportImport/export.ctl.ts

@@ -3,28 +3,17 @@ import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw';
 import { exportService } from '../../services';
 import type { Request, Response } from 'express';
 
-export async function exportBaseSchema(req: Request, res: Response) {
+export async function exportBase(req: Request, res: Response) {
   res.json(
-    await exportService.exportBaseSchema({ baseId: req.params.baseId })
+    await exportService.exportBase({ baseId: req.params.baseId, path: req.body.path })
-  );
-}
-
-export async function exportModelData(req: Request, res: Response) {
-  res.json(
-    await exportService.exportModelData({ projectId: req.params.projectId, modelId: req.params.modelId, viewId: req.params.viewId })
   );
 }
 
 const router = Router({ mergeParams: true });
 
-router.get(
+router.post(
   '/api/v1/db/meta/export/:projectId/:baseId',
-  ncMetaAclMw(exportBaseSchema, 'exportBaseSchema')
+  ncMetaAclMw(exportBase, 'exportBase')
-);
-
-router.get(
-  '/api/v1/db/meta/export/data/:projectId/:modelId/:viewId?',
-  ncMetaAclMw(exportModelData, 'exportModelData')
 );
 
 export default router;

packages/nocodb/src/lib/controllers/exportImport/import.ctl.ts

@@ -16,11 +16,24 @@ export async function importModels(req: Request, res: Response) {
   );
 }
 
+export async function importBase(req: Request, res: Response) {
+  const { body, ...rest } = req;
+  res.json(
+    await importService.importBase({
+      user: (req as any).user,
+      projectId: req.params.projectId,
+      baseId: req.params.baseId,
+      src: body.src,
+      req: rest,
+    })
+  );
+}
+
 const router = Router({ mergeParams: true });
 
 router.post(
   '/api/v1/db/meta/import/:projectId/:baseId',
-  ncMetaAclMw(importModels, 'importModels')
+  ncMetaAclMw(importBase, 'importBase')
 );
 
 export default router;

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts

@@ -2066,9 +2066,11 @@ class BaseModelSqlv2 {
     {
       chunkSize: _chunkSize = 100,
       cookie,
+      foreign_key_checks = true,
     }: {
       chunkSize?: number;
       cookie?: any;
+      foreign_key_checks?: boolean;
     } = {}
   ) {
     try {
@@ -2090,17 +2092,37 @@ class BaseModelSqlv2 {
       // refer : https://www.sqlite.org/limits.html
       const chunkSize = this.isSqlite ? 10 : _chunkSize;
 
+      const trx = await this.dbDriver.transaction();
+
+      if (!foreign_key_checks) {
+        if (this.isPg) {
+          await trx.raw('set session_replication_role to replica;');
+        } else if (this.isMySQL) {
+          await trx.raw('SET foreign_key_checks = 0;');
+        }
+      }
+
       const response =
         this.isPg || this.isMssql
-          ? await this.dbDriver
+          ? await trx
               .batchInsert(this.tnPath, insertDatas, chunkSize)
               .returning(this.model.primaryKey?.column_name)
-          : await this.dbDriver.batchInsert(
+          : await trx.batchInsert(
               this.tnPath,
               insertDatas,
               chunkSize
             );
 
+      if (!foreign_key_checks) {
+        if (this.isPg) {
+          await trx.raw('set session_replication_role to origin;');
+        } else if (this.isMySQL) {
+          await trx.raw('SET foreign_key_checks = 1;');
+        }
+      }
+
+      await trx.commit();
+
       await this.afterBulkInsert(insertDatas, this.dbDriver, cookie);
 
       return response;
@@ -2719,7 +2741,7 @@ class BaseModelSqlv2 {
     await this.afterInsert(response, this.dbDriver, cookie);
     await this.afterAddChild(rowId, childId, cookie);
   }
-
+  
   public async afterAddChild(rowId, childId, req): Promise<void> {
     await Audit.insert({
       fk_model_id: this.model.id,

packages/nocodb/src/lib/services/dbData/bulkData.ts

@@ -38,12 +38,14 @@ export async function bulkDataInsert(
   param: PathParams & {
     body: any;
     cookie: any;
+    chunkSize?: number;
+    foreign_key_checks?: boolean;
   }
 ) {
   return await executeBulkOperation({
     ...param,
     operation: 'bulkInsert',
-    options: [param.body, { cookie: param.cookie }],
+    options: [param.body, { cookie: param.cookie, foreign_key_checks: param.foreign_key_checks, chunkSize: param.chunkSize }],
   });
 }
 

packages/nocodb/src/lib/services/exportImport/export.svc.ts

@@ -1,8 +1,12 @@
 import { NcError } from './../../meta/helpers/catchError';
-import { ViewTypes } from 'nocodb-sdk';
+import { UITypes, ViewTypes } from 'nocodb-sdk';
-import { Project, Base, Model } from '../../models';
+import { Project, Base, Model, View, LinkToAnotherRecordColumn } from '../../models';
 import { dataService } from '..';
 import { getViewAndModelByAliasOrId } from '../dbData/helpers';
+import { Readable } from 'stream';
+import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2';
+import { unparse } from 'papaparse';
+import { IStorageAdapterV2 } from 'nc-plugin';
 
 /*
   {
@@ -24,6 +28,22 @@ import { getViewAndModelByAliasOrId } from '../dbData/helpers';
   }
 */
 
+async function generateBaseIdMap(base: Base, idMap: Map<string, string>) {
+  idMap.set(base.project_id, base.project_id);
+  idMap.set(base.id, `${base.project_id}::${base.id}`);
+  const models = await base.getModels();
+
+  for (const md of models) {
+    idMap.set(md.id, `${base.project_id}::${base.id}::${md.id}`);
+    await md.getColumns();
+    for (const column of md.columns) {
+      idMap.set(column.id, `${idMap.get(md.id)}::${column.id}`);
+    }
+  }
+
+  return models;
+}
+
 async function serializeModels(param: { modelId: string[] }) {
   const serializedModels = [];
 
@@ -49,31 +69,13 @@ async function serializeModels(param: { modelId: string[] }) {
     if (!fndBase) bases.push(base);
 
     if (!modelsMap.has(base.id)) {
-      const all_models = await base.getModels();
+      modelsMap.set(base.id, await generateBaseIdMap(base, idMap));
-
-      for (const md of all_models) {
-        idMap.set(md.id, `${project.id}::${base.id}::${md.id}`);
-        await md.getColumns();
-        for (const column of md.columns) {
-          idMap.set(column.id, `${idMap.get(md.id)}::${column.id}`);
-        }
-      }
-
-      modelsMap.set(base.id, all_models);
     }
 
-    idMap.set(project.id, project.id);
-    idMap.set(base.id, `${project.id}::${base.id}`);
-    idMap.set(model.id, `${idMap.get(base.id)}::${model.id}`);
-
     await model.getColumns();
     await model.getViews();
 
     for (const column of model.columns) {
-      idMap.set(
-        column.id,
-        `${idMap.get(model.id)}::${column.id}`
-      );
       await column.getColOptions();
       if (column.colOptions) {
         for (const [k, v] of Object.entries(column.colOptions)) {
@@ -248,6 +250,181 @@ async function serializeModels(param: { modelId: string[] }) {
   return serializedModels;
 }
 
+async function exportModelData(param: {
+  storageAdapter: IStorageAdapterV2;
+  path: string;
+  projectId: string;
+  modelId: string;
+  viewId?: string;
+}) {
+  const { model, view } = await getViewAndModelByAliasOrId({
+    projectName: param.projectId,
+    tableName: param.modelId,
+    viewName: param.viewId,
+  });
+
+  await model.getColumns();
+
+  const hasLink = model.columns.some((c) => c.uidt === UITypes.LinkToAnotherRecord && c.colOptions?.type === 'mm');
+
+  const pkMap = new Map<string, string>();
+
+  for (const column of model.columns.filter((c) => c.uidt === UITypes.LinkToAnotherRecord && c.colOptions?.type !== 'hm')) {
+    const relatedTable = await (
+      (await column.getColOptions()) as LinkToAnotherRecordColumn
+    ).getRelatedTable();
+
+    await relatedTable.getColumns();
+
+    pkMap.set(column.id, relatedTable.primaryKey.title);
+  }
+
+  const readableStream = new Readable({
+    read() {},
+  });
+
+  const readableLinkStream = new Readable({
+    read() {},
+  });
+
+  readableStream.setEncoding('utf8');
+
+  readableLinkStream.setEncoding('utf8');
+
+  const storageAdapter = param.storageAdapter;
+
+  const uploadPromise = storageAdapter.fileCreateByStream(
+    `${param.path}/${model.id}.csv`,
+    readableStream
+  );
+
+  const uploadLinkPromise = hasLink
+    ? storageAdapter.fileCreateByStream(
+        `${param.path}/${model.id}_links.csv`,
+        readableLinkStream
+      )
+    : Promise.resolve();
+
+  const limit = 100;
+  let offset = 0;
+
+  const primaryKey = model.columns.find((c) => c.pk);
+
+  const formatData = (data: any) => {
+    const linkData = [];
+    for (const row of data) {
+      const pkValue = primaryKey ? row[primaryKey.title] : undefined;
+      const linkRow = {};
+      for (const [k, v] of Object.entries(row)) {
+        const col = model.columns.find((c) => c.title === k);
+        if (col) {
+          if (col.pk) linkRow['pk'] = pkValue;
+          const colId = `${col.project_id}::${col.base_id}::${col.fk_model_id}::${col.id}`;
+          switch(col.uidt) {
+            case UITypes.LinkToAnotherRecord:
+              if (col.system || col.colOptions.type === 'hm') break;
+              const pkList = [];
+
+              const links = Array.isArray(v) ? v : [v];
+
+              for (const link of links) {
+                if (link) {
+                  for (const [k, val] of Object.entries(link)) {
+                    if (k === pkMap.get(col.id)) {
+                      pkList.push(val);
+                    }
+                  }
+                }
+              }
+
+              if (col.colOptions.type === 'mm') {
+                linkRow[colId] = pkList.join(',');
+              } else {
+                row[colId] = pkList[0];
+              }
+              break;
+            case UITypes.Attachment:
+              try {
+                row[colId] = JSON.stringify(v);
+              } catch (e) {
+                row[colId] = v;
+              }
+              break;
+            case UITypes.ForeignKey:
+            case UITypes.Formula:
+            case UITypes.Lookup:
+            case UITypes.Rollup:
+            case UITypes.Rating:
+            case UITypes.Barcode:
+              // skip these types
+              break;
+            default:
+              row[colId] = v;
+              break;
+          }
+          delete row[k];
+        }
+      }
+      linkData.push(linkRow);
+    }
+    return { data, linkData };
+  }
+
+  try {
+    await recursiveRead(formatData, readableStream, readableLinkStream, model, view, offset, limit, true);
+    await uploadPromise;
+    await uploadLinkPromise;
+  } catch (e) {
+    await storageAdapter.fileDelete(`${param.path}/${model.id}.csv`);
+    await storageAdapter.fileDelete(`${param.path}/${model.id}_links.csv`);
+    console.error(e);
+    throw e;
+  }
+
+  return true;
+}
+
+async function recursiveRead(
+  formatter: Function,
+  stream: Readable,
+  linkStream: Readable,
+  model: Model,
+  view: View,
+  offset: number,
+  limit: number,
+  header = false
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    dataService
+      .getDataList({ model, view, query: { limit, offset } })
+      .then((result) => {
+        try {
+          if (!header) {
+            stream.push('\r\n');
+            linkStream.push('\r\n');
+          }
+          const { data, linkData } = formatter(result.list);
+          stream.push(unparse(data, { header }));
+          linkStream.push(unparse(linkData, { header }));
+          if (result.pageInfo.isLastPage) {
+            stream.push(null);
+            linkStream.push(null);
+            resolve();
+          } else {
+            recursiveRead(formatter, stream, linkStream, model, view, offset + limit, limit).then(resolve);
+          }
+        } catch (e) {
+          reject(e);
+        }
+      });
+  });
+}
+
+function clearPrefix(text: string, prefix?: string) {
+  if (!prefix || prefix.length === 0) return text;
+  return text.replace(new RegExp(`^${prefix}_?`), '');
+}
+
 export async function exportBaseSchema(param: { baseId: string }) {
   const base = await Base.get(param.baseId);
 
@@ -255,7 +432,7 @@ export async function exportBaseSchema(param: { baseId: string }) {
 
   const project = await Project.get(base.project_id);
 
-  const models = (await base.getModels()).filter((m) => !m.mm);
+  const models = (await base.getModels()).filter((m) => !m.mm && m.type === 'table');
 
   const exportedModels = await serializeModels({ modelId: models.map(m => m.id) });
 
@@ -264,7 +441,53 @@ export async function exportBaseSchema(param: { baseId: string }) {
   return exportData;
 }
 
-const clearPrefix = (text: string, prefix?: string) => {
+export async function exportBase(param: { path: string; baseId: string }) {
-  if (!prefix || prefix.length === 0) return text;
+  const base = await Base.get(param.baseId);
-  return text.replace(new RegExp(`^${prefix}_?`), '');
+
+  if (!base) return NcError.badRequest(`Base not found for id '${param.baseId}'`);
+
+  const project = await Project.get(base.project_id);
+
+  const models = (await base.getModels()).filter((m) => !m.mm && m.type === 'table');
+
+  const exportedModels = await serializeModels({ modelId: models.map(m => m.id) });
+
+  const exportData = { id: `${project.id}::${base.id}`, entity: 'base', models: exportedModels };
+
+  const storageAdapter = await NcPluginMgrv2.storageAdapter();
+
+  const destPath = `export/${project.id}/${base.id}/${param.path}/schema.json`;
+
+  try {
+
+    const readableStream = new Readable({
+      read() {},
+    });
+
+    readableStream.setEncoding('utf8');
+
+    readableStream.push(JSON.stringify(exportData));
+
+    readableStream.push(null);
+
+    await storageAdapter.fileCreateByStream(
+      destPath,
+      readableStream
+    );
+
+    for (const model of models) {
+      await exportModelData({
+        storageAdapter,
+        path: `export/${project.id}/${base.id}/${param.path}/data`,
+        projectId: project.id,
+        modelId: model.id,
+      });
+    }
+
+  } catch (e) {
+    console.error(e);
+    return NcError.internalServerError('Error while exporting base');
+  }
+
+  return true;
 }

packages/nocodb/src/lib/services/exportImport/import.svc.ts

@@ -1,14 +1,16 @@
 import type { ViewCreateReqType } from 'nocodb-sdk';
 import { UITypes, ViewTypes } from 'nocodb-sdk';
-import { tableService, gridViewService, filterService, viewColumnService, gridViewColumnService, sortService, formViewService, galleryViewService, kanbanViewService, formViewColumnService, columnService } from '..';
+import { tableService, gridViewService, filterService, viewColumnService, gridViewColumnService, sortService, formViewService, galleryViewService, kanbanViewService, formViewColumnService, columnService, bulkDataService } from '..';
 import { NcError } from '../../meta/helpers/catchError';
-import { Project, Base, User, View, Model } from '../../models';
+import { Project, Base, User, View, Model, Column, LinkToAnotherRecordColumn } from '../../models';
+import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2';
+import papaparse from 'papaparse';
 
 export async function importModels(param: {
   user: User;
   projectId: string;
   baseId: string;
-  data: { model: any; views: any[] }[];
+  data: { models: { model: any; views: any[] }[] } | { model: any; views: any[] }[];
   req: any;
 }) {
 
@@ -25,6 +27,8 @@ export async function importModels(param: {
 
   const tableReferences = new Map<string, Model>();
   const linkMap = new Map<string, string>();
+  
+  param.data = Array.isArray(param.data) ? param.data : param.data.models;
 
   // create tables with static columns
   for (const data of param.data) {
@@ -83,7 +87,7 @@ export async function importModels(param: {
             if (!linkMap.has(colOptions.fk_mm_model_id)) {
               // delete col.column_name as it is not required and will cause ajv error (null for LTAR)
               delete col.column_name;
-
+              
               const freshModelData = await columnService.columnAdd({
                 tableId: table.id,
                 column: withoutId({
@@ -118,14 +122,16 @@ export async function importModels(param: {
                 if (nColumn?.colOptions?.fk_mm_model_id === linkMap.get(colOptions.fk_mm_model_id) && nColumn.id !== idMap.get(col.id)) {
                   idMap.set(childColumn.id, nColumn.id);
 
-                  await columnService.columnUpdate({
+                  if (nColumn.title !== childColumn.title) {
-                    columnId: nColumn.id,
+                    await columnService.columnUpdate({
-                    column: {
+                      columnId: nColumn.id,
-                      ...nColumn,
+                      column: {
-                      column_name: childColumn.title,
+                        ...nColumn,
-                      title: childColumn.title,
+                        column_name: childColumn.title,
-                    },
+                        title: childColumn.title,
-                  });
+                      },
+                    });
+                  }
                   break;
                 }
               }
@@ -153,7 +159,8 @@ export async function importModels(param: {
             for (const nColumn of freshModelData.columns) {
               if (nColumn.title === col.title) {
                 idMap.set(col.id, nColumn.id);
-                linkMap.set(colOptions.fk_index_name, nColumn.colOptions.fk_index_name);
+                idMap.set(colOptions.fk_parent_column_id, nColumn.colOptions.fk_parent_column_id);
+                idMap.set(colOptions.fk_child_column_id, nColumn.colOptions.fk_child_column_id);
                 break;
               }
             }
@@ -162,20 +169,31 @@ export async function importModels(param: {
 
             if (colOptions.fk_related_model_id !== modelData.id) await childModel.getColumns();
 
-            const childColumn = param.data.find(a => a.model.id === colOptions.fk_related_model_id).model.columns.find(a => a.colOptions?.fk_index_name === colOptions.fk_index_name && a.id !== col.id);
+            const childColumn = param.data
+              .find((a) => a.model.id === colOptions.fk_related_model_id)
+              .model.columns.find(
+                (a) =>
+                  a.colOptions?.fk_parent_column_id ===
+                    colOptions.fk_parent_column_id &&
+                  a.colOptions?.fk_child_column_id ===
+                    colOptions.fk_child_column_id &&
+                  a.id !== col.id
+              );
 
             for (const nColumn of childModel.columns) {
-              if (nColumn?.colOptions?.fk_index_name === linkMap.get(colOptions.fk_index_name) && nColumn.id !== idMap.get(col.id)) {
+              if (nColumn.id !== idMap.get(col.id) && nColumn.colOptions?.fk_parent_column_id === idMap.get(colOptions.fk_parent_column_id) && nColumn.colOptions?.fk_child_column_id === idMap.get(colOptions.fk_child_column_id)) {
                 idMap.set(childColumn.id, nColumn.id);
 
-                await columnService.columnUpdate({
+                if (nColumn.title !== childColumn.title) {
-                  columnId: nColumn.id,
+                  await columnService.columnUpdate({
-                  column: {
+                    columnId: nColumn.id,
-                    ...nColumn,
+                    column: {
-                    column_name: childColumn.title,
+                      ...nColumn,
-                    title: childColumn.title,
+                      column_name: childColumn.title,
-                  },
+                      title: childColumn.title,
-                });
+                    },
+                  });
+                }
                 break;
               }
             }
@@ -385,6 +403,8 @@ export async function importModels(param: {
       }
     }
   }
+
+  return idMap;
 }
 
 async function createView(idMap: Map<string, string>, md: Model, vw: Partial<View>, views: View[]): Promise<View> {
@@ -529,9 +549,272 @@ function getParentIdentifier(id: string) {
   arr.pop();
   return arr.join('::');
 }
-/*
+
 function getEntityIdentifier(id: string) {
   const arr = id.split('::');
   return arr.pop();
 }
-*/
+
+function findWithIdentifier(map: Map<string, any>, id: string) {
+  for (const key of map.keys()) {
+    if (getEntityIdentifier(key) === id) {
+      return map.get(key);
+    }
+  }
+  return undefined;
+}
+
+export async function importBase(param: {
+  user: User;
+  projectId: string;
+  baseId: string;
+  src: { type: 'local' | 'url' | 'file'; path?: string; url?: string; file?: any };
+  req: any;
+}) {
+  const { user, projectId, baseId, src, req } = param;
+
+  let start = process.hrtime();
+
+  let elapsed_time = function(label: string){
+      const elapsedS = (process.hrtime(start)[0]).toFixed(3);
+      const elapsedMs = process.hrtime(start)[1] / 1000000;
+      console.log(`${label}: ${elapsedS}s ${elapsedMs}ms`);
+      start = process.hrtime();
+  }
+
+  switch (src.type) {
+    case 'local':
+      const path = src.path.replace(/\/$/, '');
+
+      const storageAdapter = await NcPluginMgrv2.storageAdapter();
+
+      try {
+        const schema = JSON.parse(await storageAdapter.fileRead(`${path}/schema.json`));
+
+        elapsed_time('read schema');
+
+        // store fk_mm_model_id (mm) to link once
+        const handledLinks = [];
+
+        const idMap = await importModels({
+          user,
+          projectId,
+          baseId,
+          data: schema,
+          req,
+        });
+
+        elapsed_time('import models');
+
+        if (idMap) {
+          const files = await storageAdapter.getDirectoryList(`${path}/data`);
+          const dataFiles = files.filter((file) => !file.match(/_links\.csv$/));
+          const linkFiles = files.filter((file) => file.match(/_links\.csv$/));
+        
+          for (const file of dataFiles) {
+            const readStream = await storageAdapter.fileReadByStream(
+              `${path}/data/${file}`
+            );
+            
+            const headers: string[] = [];
+            let chunk = [];
+
+            const modelId = findWithIdentifier(
+              idMap,
+              file.replace(/\.csv$/, '')
+            );
+
+            const model = await Model.get(modelId);
+
+            console.log(`Importing ${model.title}...`);
+
+            await new Promise(async (resolve) => {
+              papaparse.parse(readStream, {
+                newline: '\r\n',
+                step: async function (results, parser) {
+                  if (!headers.length) {
+                    parser.pause();
+                    for (const header of results.data) {
+                      const id = idMap.get(header);
+                      if (id) {
+                        const col = await Column.get({
+                          base_id: baseId,
+                          colId: id,
+                        });
+                        if (col.colOptions?.type === 'bt') {
+                          const childCol = await Column.get({
+                            base_id: baseId,
+                            colId: col.colOptions.fk_child_column_id,
+                          });
+                          headers.push(childCol.title);
+                        } else {
+                          headers.push(col.title);
+                        }
+                        
+                      } else {
+                          console.log(header);
+                      }
+                    }
+                    parser.resume();
+                  } else {
+                    if (results.errors.length === 0) {
+                      const row = {};
+                      for (let i = 0; i < headers.length; i++) {
+                        if (results.data[i] !== '') {
+                          row[headers[i]] = results.data[i];
+                        }
+                      }
+                      chunk.push(row);
+                      if (chunk.length > 1000) {
+                        parser.pause();
+                        elapsed_time('before chunk');
+                        await bulkDataService.bulkDataInsert({
+                          projectName: projectId,
+                          tableName: modelId,
+                          body: chunk,
+                          cookie: null,
+                          chunkSize: 1000,
+                          foreign_key_checks: false
+                        });
+                        chunk = [];
+                        elapsed_time('after chunk');
+                        parser.resume();
+                      }
+                    }
+                  }
+                },
+                complete: async function () {
+                  if (chunk.length > 0) {
+                    elapsed_time('before chunk');
+                    await bulkDataService.bulkDataInsert({
+                      projectName: projectId,
+                      tableName: modelId,
+                      body: chunk,
+                      cookie: null,
+                      foreign_key_checks: false
+                    });
+                    chunk = [];
+                    elapsed_time('after chunk');
+                  }
+                  resolve(null);
+                },
+              });
+            });
+          }
+
+          for (const file of linkFiles) {
+            const readStream = await storageAdapter.fileReadByStream(
+              `${path}/data/${file}`
+            );
+
+            const headers: string[] = [];
+            const mmParentChild: any = {};
+            let chunk: Record<string, any[]> = {}; // colId: { rowId, childId }[]
+
+            const modelId = findWithIdentifier(
+              idMap,
+              file.replace(/_links\.csv$/, '')
+            );
+            const model = await Model.get(modelId);
+            
+            let pkIndex = -1;
+
+            console.log(`Linking ${model.title}...`);
+
+            await new Promise(async (resolve) => {
+              papaparse.parse(readStream, {
+                newline: '\r\n',
+                step: async function (results, parser) {
+                  if (!headers.length) {
+                    parser.pause();
+                    for (const header of results.data) {
+                      if (header === 'pk') {
+                        headers.push(null);
+                        pkIndex = headers.length - 1;
+                        continue;
+                      }
+                      const id = idMap.get(header);
+                      if (id) {
+                        const col = await Column.get({
+                          base_id: baseId,
+                          colId: id,
+                        });
+                        if (
+                          col.uidt === UITypes.LinkToAnotherRecord &&
+                          col.colOptions.fk_mm_model_id &&
+                          handledLinks.includes(col.colOptions.fk_mm_model_id)
+                        ) {
+                          headers.push(null);
+                        } else {
+                          if (
+                            col.uidt === UITypes.LinkToAnotherRecord &&
+                            col.colOptions.fk_mm_model_id &&
+                            !handledLinks.includes(
+                              col.colOptions.fk_mm_model_id
+                            )
+                          ) {
+                            const colOptions = await col.getColOptions<LinkToAnotherRecordColumn>();
+
+                            const vChildCol = await colOptions.getMMChildColumn();
+                            const vParentCol = await colOptions.getMMParentColumn();
+
+                            mmParentChild[col.colOptions.fk_mm_model_id] = {
+                              parent: vParentCol.title,
+                              child: vChildCol.title,
+                            }
+
+                            handledLinks.push(col.colOptions.fk_mm_model_id);
+                          }
+                          headers.push(col.colOptions.fk_mm_model_id);
+                          chunk[col.colOptions.fk_mm_model_id] = []
+                        }
+                      }
+                    }
+                    parser.resume();
+                  } else {
+                    if (results.errors.length === 0) {
+                      for (let i = 0; i < headers.length; i++) {
+                        if (!headers[i]) continue;
+
+                        const mm = mmParentChild[headers[i]];
+
+                        for (const rel of results.data[i].split(',')) {
+                          if (rel.trim() === '') continue;
+                          chunk[headers[i]].push({ [mm.parent]: rel, [mm.child]: results.data[pkIndex] });
+                        }
+                      }
+                    }
+                  }
+                },
+                complete: async function () {
+                  for (const [k, v] of Object.entries(chunk)) {
+                    try {                      
+                      await bulkDataService.bulkDataInsert({
+                        projectName: projectId,
+                        tableName: k,
+                        body: v,
+                        cookie: null,
+                        chunkSize: 1000,
+                        foreign_key_checks: false
+                      });
+                    } catch (e) {
+                      console.log('linkError');
+                      console.log(e);
+                    }
+                  }
+                  resolve(null);
+                },
+              });
+            });
+          }
+        }
+      } catch (e) {
+        throw new Error(e);
+      }
+      break;
+    case 'url':
+      break;
+    case 'file':
+      break;
+  }
+}