From 6d51ce7bfc5c172f372c81dc08dcc007e4e9b346 Mon Sep 17 00:00:00 2001
From: mertmit <mertmit99@gmail.com>
Date: Tue, 25 Apr 2023 23:27:30 +0300
Subject: [PATCH] feat: at import background job

Signed-off-by: mertmit <mertmit99@gmail.com>
---
 .../nc-gui/components/dlg/AirtableImport.vue  |  111 +-
 packages/nc-gui/pages/index/index/index.vue   |    2 +-
 packages/nc-gui/plugins/jobs.ts               |   56 +-
 .../imports/helpers/readAndProcessData.ts     |    5 +-
 .../jobs/at-import/at-import.controller.ts    |   76 +
 .../jobs/at-import/at-import.processor.ts     | 2515 +++++++++++++++++
 .../jobs/at-import/helpers/EntityMap.ts       |  222 ++
 .../modules/jobs/at-import/helpers/fetchAT.ts |  242 ++
 .../at-import/helpers/readAndProcessData.ts   |  362 +++
 .../modules/jobs/at-import/helpers/syncMap.ts |   31 +
 .../modules/jobs/fallback-queue.service.ts    |    6 +
 .../src/modules/jobs/jobs-event.service.ts    |    1 +
 .../src/modules/jobs/jobs.gateway.ts          |   22 +-
 .../src/modules/jobs/jobs.module.ts           |    5 +-
 .../src/modules/jobs/jobs.service.ts          |    6 +-
 .../src/modules/metas/metas.module.ts         |    4 +-
 .../src/services/socket.gateway.ts            |   12 -
 17 files changed, 3565 insertions(+), 113 deletions(-)
 create mode 100644 packages/nocodb-nest/src/modules/jobs/at-import/at-import.controller.ts
 create mode 100644 packages/nocodb-nest/src/modules/jobs/at-import/at-import.processor.ts
 create mode 100644 packages/nocodb-nest/src/modules/jobs/at-import/helpers/EntityMap.ts
 create mode 100644 packages/nocodb-nest/src/modules/jobs/at-import/helpers/fetchAT.ts
 create mode 100644 packages/nocodb-nest/src/modules/jobs/at-import/helpers/readAndProcessData.ts
 create mode 100644 packages/nocodb-nest/src/modules/jobs/at-import/helpers/syncMap.ts

diff --git a/packages/nc-gui/components/dlg/AirtableImport.vue b/packages/nc-gui/components/dlg/AirtableImport.vue
index a86bc63e32..15c0b15e4c 100644
--- a/packages/nc-gui/components/dlg/AirtableImport.vue
+++ b/packages/nc-gui/components/dlg/AirtableImport.vue
@@ -1,6 +1,4 @@
 <script setup lang="ts">
-import type { Socket } from 'socket.io-client'
-import io from 'socket.io-client'
 import type { Card as AntCard } from 'ant-design-vue'
 import {
   Form,
@@ -10,7 +8,6 @@ import {
   iconMap,
   message,
   nextTick,
-  onBeforeUnmount,
   onMounted,
   ref,
   storeToRefs,
@@ -31,7 +28,7 @@ const { appInfo } = $(useGlobal())
 
 const baseURL = appInfo.ncSiteUrl
 
-const { $state } = useNuxtApp()
+const { $state, $jobs } = useNuxtApp()
 
 const projectStore = useProject()
 
@@ -49,8 +46,6 @@ const logRef = ref<typeof AntCard>()
 
 const enableAbort = ref(false)
 
-let socket: Socket | null
-
 const syncSource = ref({
   id: '',
   type: 'Airtable',
@@ -72,6 +67,35 @@ const syncSource = ref({
   },
 })
 
+const pushProgress = async (message: string, status: 'completed' | 'failed' | 'progress') => {
+  progress.value.push({ msg: message, status })
+
+  await nextTick(() => {
+    const container: HTMLDivElement = logRef.value?.$el?.firstElementChild
+    if (!container) return
+    container.scrollTop = container.scrollHeight
+  })
+}
+
+const onSubscribe = () => {
+  step.value = 2
+}
+
+const onStatus = async (status: 'active' | 'completed' | 'failed' | 'refresh', error?: any) => {
+  if (status === 'completed') {
+    showGoToDashboardButton.value = true
+    await loadTables()
+    pushProgress('Done!', status)
+    // TODO: add tab of the first table
+  } else if (status === 'failed') {
+    pushProgress(error, status)
+  }
+}
+
+const onLog = (data: { message: string }) => {
+  pushProgress(data.message, 'progress')
+}
+
 const validators = computed(() => ({
   'details.apiKey': [fieldRequiredValidator()],
   'details.syncSourceUrlOrId': [fieldRequiredValidator()],
@@ -130,7 +154,7 @@ async function loadSyncSrc() {
     srcs[0].details = srcs[0].details || {}
     syncSource.value = migrateSync(srcs[0])
     syncSource.value.details.syncSourceUrlOrId = srcs[0].details.shareId
-    socket?.emit('subscribe', syncSource.value.id)
+    $jobs.subscribe({ syncId: syncSource.value.id }, onSubscribe, onStatus, onLog)
   } else {
     syncSource.value = {
       id: '',
@@ -161,11 +185,8 @@ async function sync() {
       baseURL,
       method: 'POST',
       headers: { 'xc-auth': $state.token.value as string },
-      params: {
-        id: socket?.id,
-      },
     })
-    socket?.emit('subscribe', syncSource.value.id)
+    $jobs.subscribe({ syncId: syncSource.value.id }, onSubscribe, onStatus, onLog)
   } catch (e: any) {
     message.error(await extractSdkResponseErrorMsg(e))
   }
@@ -183,9 +204,6 @@ async function abort() {
           baseURL,
           method: 'POST',
           headers: { 'xc-auth': $state.token.value as string },
-          params: {
-            id: socket?.id,
-          },
         })
         step.value = 1
       } catch (e: any) {
@@ -223,67 +241,12 @@ watch(
 )
 
 onMounted(async () => {
-  socket = io(new URL(baseURL, window.location.href.split(/[?#]/)[0]).href, {
-    extraHeaders: { 'xc-auth': $state.token.value as string },
-  })
-
-  socket.on('progress', async (d: Record<string, any>) => {
-    progress.value.push(d)
-
-    await nextTick(() => {
-      const container: HTMLDivElement = logRef.value?.$el?.firstElementChild
-      if (!container) return
-      container.scrollTop = container.scrollHeight
-    })
-
-    if (d.status === 'COMPLETED') {
-      showGoToDashboardButton.value = true
-      await loadTables()
-      // TODO: add tab of the first table
-    }
-  })
-
-  socket.on('disconnect', () => {
-    console.log('socket disconnected')
-    const rcInterval = setInterval(() => {
-      if (socket?.connected) {
-        clearInterval(rcInterval)
-        socket?.emit('subscribe', syncSource.value.id)
-      } else {
-        socket?.connect()
-      }
-    }, 2000)
-  })
-
-  socket.on('job', () => {
-    step.value = 2
-  })
-
-  // connect event does not provide data
-  socket.on('connect', () => {
-    console.log('socket connected')
-    if (syncSource.value.id) {
-      socket?.emit('subscribe', syncSource.value.id)
-    }
-  })
-
-  socket?.io.on('reconnect', () => {
-    console.log('socket reconnected')
-    if (syncSource.value.id) {
-      socket?.emit('subscribe', syncSource.value.id)
-    }
-  })
+  if (syncSource.value.id) {
+    $jobs.subscribe({ syncId: syncSource.value.id }, onSubscribe, onStatus, onLog)
+  }
 
   await loadSyncSrc()
 })
-
-onBeforeUnmount(() => {
-  if (socket) {
-    socket.off('disconnect')
-    socket.disconnect()
-    socket.removeAllListeners()
-  }
-})
 </script>
 
 <template>
@@ -407,7 +370,7 @@ onBeforeUnmount(() => {
 
         <a-card ref="logRef" :body-style="{ backgroundColor: '#000000', height: '400px', overflow: 'auto' }">
           <div v-for="({ msg, status }, i) in progress" :key="i">
-            <div v-if="status === 'FAILED'" class="flex items-center">
+            <div v-if="status === 'failed'" class="flex items-center">
               <component :is="iconMap.closeCircle" class="text-red-500" />
 
               <span class="text-red-500 ml-2">{{ msg }}</span>
@@ -424,7 +387,7 @@ onBeforeUnmount(() => {
             v-if="
               !progress ||
               !progress.length ||
-              (progress[progress.length - 1].status !== 'COMPLETED' && progress[progress.length - 1].status !== 'FAILED')
+              (progress[progress.length - 1].status !== 'completed' && progress[progress.length - 1].status !== 'failed')
             "
             class="flex items-center"
           >
diff --git a/packages/nc-gui/pages/index/index/index.vue b/packages/nc-gui/pages/index/index/index.vue
index 18b4732536..c00946c751 100644
--- a/packages/nc-gui/pages/index/index/index.vue
+++ b/packages/nc-gui/pages/index/index/index.vue
@@ -95,7 +95,7 @@ const duplicateProject = (project: ProjectType) => {
 
         await loadProjects()
 
-        $jobs.subscribe(jobData.name, jobData.id, async (data: { status: string }) => {
+        $jobs.subscribe({ name: jobData.name, id: jobData.id }, null, async (data: { status: string }) => {
           if (data.status === 'completed') {
             await loadProjects()
           } else if (data.status === 'failed') {
diff --git a/packages/nc-gui/plugins/jobs.ts b/packages/nc-gui/plugins/jobs.ts
index b672da60d1..ceede03259 100644
--- a/packages/nc-gui/plugins/jobs.ts
+++ b/packages/nc-gui/plugins/jobs.ts
@@ -6,6 +6,7 @@ export default defineNuxtPlugin(async (nuxtApp) => {
   const { appInfo } = $(useGlobal())
 
   let socket: Socket | null = null
+  let messageIndex = 0
 
   const init = async (token: string) => {
     try {
@@ -28,27 +29,58 @@ export default defineNuxtPlugin(async (nuxtApp) => {
     await init(nuxtApp.$state.token.value)
   }
 
+  const send = (name: string, data: any) => {
+    if (socket) {
+      const _id = messageIndex++
+      socket.emit(name, { _id, data })
+      return _id
+    }
+  }
+
   const jobs = {
-    subscribe(name: string, id: string, cb: (data: any) => void) {
-      if (socket) {
-        socket.emit('subscribe', { name, id })
-        const tempFn = (data: any) => {
-          if (data.id === id && data.name === name) {
-            cb(data)
-            if (data.status === 'completed' || data.status === 'failed') {
-              socket?.off('status', tempFn)
-            }
+    subscribe(
+      job: { id: string; name: string } | any,
+      subscribedCb?: () => void,
+      statusCb?: (status: 'active' | 'completed' | 'failed' | 'refresh', error?: any) => void,
+      logCb?: (data: { message: string }) => void,
+    ) {
+      const logFn = (data: { id: string; name: string; data: { message: string } }) => {
+        if (data.id === job.id) {
+          if (logCb) logCb(data.data)
+        }
+      }
+      const statusFn = (data: any) => {
+        if (data.id === job.id) {
+          if (statusCb) statusCb(data.status, data.error)
+          if (data.status === 'completed' || data.status === 'failed') {
+            socket?.off('status', statusFn)
+            socket?.off('log', logFn)
+          }
+        }
+      }
+
+      const _id = send('subscribe', job)
+
+      const subscribeFn = (data: { _id: number; name: string; id: string }) => {
+        if (data._id === _id) {
+          if (data.id !== job.id || data.name !== job.name) {
+            job.id = data.id
+            job.name = data.name
           }
+          if (subscribedCb) subscribedCb()
+          socket?.on('log', logFn)
+          socket?.on('status', statusFn)
+          socket?.off('subscribed', subscribeFn)
         }
-        socket.on('status', tempFn)
       }
+      socket?.on('subscribed', subscribeFn)
     },
     getStatus(name: string, id: string): Promise<string> {
       return new Promise((resolve) => {
         if (socket) {
-          socket.emit('status', { name, id })
+          const _id = send('status', { name, id })
           const tempFn = (data: any) => {
-            if (data.id === id && data.name === name) {
+            if (data._id === _id) {
               resolve(data.status)
               socket?.off('status', tempFn)
             }
diff --git a/packages/nocodb-nest/src/controllers/imports/helpers/readAndProcessData.ts b/packages/nocodb-nest/src/controllers/imports/helpers/readAndProcessData.ts
index 45cbc3bd2e..7de16fad7e 100644
--- a/packages/nocodb-nest/src/controllers/imports/helpers/readAndProcessData.ts
+++ b/packages/nocodb-nest/src/controllers/imports/helpers/readAndProcessData.ts
@@ -1,7 +1,8 @@
+/* eslint-disable no-async-promise-executor */
 import { RelationTypes, UITypes } from 'nocodb-sdk';
 import EntityMap from './EntityMap';
-import type { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
-import type { TablesService } from '../../../services/tables.service';
+import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
+import type { TablesService } from '../../../../services/tables.service';
 // @ts-ignore
 import type { AirtableBase } from 'airtable/lib/airtable_base';
 import type { TableType } from 'nocodb-sdk';
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/at-import.controller.ts b/packages/nocodb-nest/src/modules/jobs/at-import/at-import.controller.ts
new file mode 100644
index 0000000000..51ec6657ff
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/at-import.controller.ts
@@ -0,0 +1,76 @@
+import { InjectQueue } from '@nestjs/bull';
+import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common';
+import { GlobalGuard } from 'src/guards/global/global.guard';
+import { ExtractProjectIdMiddleware } from 'src/middlewares/extract-project-id/extract-project-id.middleware';
+import { Queue } from 'bull';
+import { SyncSource } from 'src/models';
+import { NcError } from 'src/helpers/catchError';
+import { QueueService } from '../fallback-queue.service';
+import { JobsService } from '../jobs.service';
+
+@Controller()
+@UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
+export class AtImportController {
+  activeQueue;
+  constructor(
+    @InjectQueue('jobs') private readonly jobsQueue: Queue,
+    private readonly fallbackQueueService: QueueService,
+    private readonly jobsService: JobsService,
+  ) {
+    this.activeQueue = process.env.NC_REDIS_URL
+      ? this.jobsQueue
+      : this.fallbackQueueService;
+  }
+
+  @Post('/api/v1/db/meta/import/airtable')
+  @HttpCode(200)
+  async importAirtable(@Request() req) {
+    const job = await this.activeQueue.add('at-import', {
+      ...req.body,
+    });
+
+    return { id: job.id, name: job.name };
+  }
+
+  @Post('/api/v1/db/meta/syncs/:syncId/trigger')
+  @HttpCode(200)
+  async triggerSync(@Request() req) {
+    const jobs = await this.jobsService.jobList('at-import');
+    const fnd = jobs.find((j) => j.data.syncId === req.params.syncId);
+
+    if (fnd) {
+      NcError.badRequest('Sync already in progress');
+    }
+
+    const syncSource = await SyncSource.get(req.params.syncId);
+
+    const user = await syncSource.getUser();
+
+    // Treat default baseUrl as siteUrl from req object
+    let baseURL = (req as any).ncSiteUrl;
+
+    // if environment value avail use it
+    // or if it's docker construct using `PORT`
+    if (process.env.NC_DOCKER) {
+      baseURL = `http://localhost:${process.env.PORT || 8080}`;
+    }
+
+    const job = await this.activeQueue.add('at-import', {
+      syncId: req.params.syncId,
+      ...(syncSource?.details || {}),
+      projectId: syncSource.project_id,
+      baseId: syncSource.base_id,
+      authToken: '',
+      baseURL,
+      user: user,
+    });
+
+    return { id: job.id, name: job.name };
+  }
+
+  @Post('/api/v1/db/meta/syncs/:syncId/abort')
+  @HttpCode(200)
+  async abortImport(@Request() req) {
+    return {};
+  }
+}
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/at-import.processor.ts b/packages/nocodb-nest/src/modules/jobs/at-import/at-import.processor.ts
new file mode 100644
index 0000000000..0ab6e7e30c
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/at-import.processor.ts
@@ -0,0 +1,2515 @@
+import { promisify } from 'util';
+import { UITypes } from 'nocodb-sdk';
+import Airtable from 'airtable';
+import jsonfile from 'jsonfile';
+import hash from 'object-hash';
+import { T } from 'nc-help';
+import dayjs from 'dayjs';
+import utc from 'dayjs/plugin/utc';
+import tinycolor from 'tinycolor2';
+import { Process, Processor } from '@nestjs/bull';
+import { Job } from 'bull';
+import extractRolesObj from '../../../utils/extractRolesObj';
+import { AttachmentsService } from '../../../services/attachments.service';
+import { ColumnsService } from '../../../services/columns.service';
+import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
+import { FiltersService } from '../../../services/filters.service';
+import { FormColumnsService } from '../../../services/form-columns.service';
+import { GalleriesService } from '../../../services/galleries.service';
+import { GridsService } from '../../../services/grids.service';
+import { ProjectUsersService } from '../../../services/project-users/project-users.service';
+import { ProjectsService } from '../../../services/projects.service';
+import { SortsService } from '../../../services/sorts.service';
+import { TablesService } from '../../../services/tables.service';
+import { ViewColumnsService } from '../../../services/view-columns.service';
+import { ViewsService } from '../../../services/views.service';
+import { FormsService } from '../../../services/forms.service';
+import { JobsEventService } from '../jobs-event.service';
+import FetchAT from './helpers/fetchAT';
+import { importData, importLTARData } from './helpers/readAndProcessData';
+import EntityMap from './helpers/EntityMap';
+import type { UserType } from 'nocodb-sdk';
+
+const writeJsonFileAsync = promisify(jsonfile.writeFile);
+
+dayjs.extend(utc);
+
+const selectColors = {
+  // normal
+  blue: '#cfdfff',
+  cyan: '#d0f0fd',
+  teal: '#c2f5e9',
+  green: '#d1f7c4',
+  orange: '#fee2d5',
+  yellow: '#ffeab6',
+  red: '#ffdce5',
+  pink: '#ffdaf6',
+  purple: '#ede2fe',
+  gray: '#eee',
+  // medium
+  blueMedium: '#9cc7ff',
+  cyanMedium: '#77d1f3',
+  tealMedium: '#72ddc3',
+  greenMedium: '#93e088',
+  orangeMedium: '#ffa981',
+  yellowMedium: '#ffd66e',
+  redMedium: '#ff9eb7',
+  pinkMedium: '#f99de2',
+  purpleMedium: '#cdb0ff',
+  grayMedium: '#ccc',
+  // dark
+  blueDark: '#2d7ff9',
+  cyanDark: '#18bfff',
+  tealDark: '#20d9d2',
+  greenDark: '#20c933',
+  orangeDark: '#ff6f2c',
+  yellowDark: '#fcb400',
+  redDark: '#f82b60',
+  pinkDark: '#ff08c2',
+  purpleDark: '#8b46ff',
+  grayDark: '#666',
+  // darker
+  blueDarker: '#2750ae',
+  cyanDarker: '#0b76b7',
+  tealDarker: '#06a09b',
+  greenDarker: '#338a17',
+  orangeDarker: '#d74d26',
+  yellowDarker: '#b87503',
+  redDarker: '#ba1e45',
+  pinkDarker: '#b2158b',
+  purpleDarker: '#6b1cb0',
+  grayDarker: '#444',
+};
+
+@Processor('jobs')
+export class AtImportProcessor {
+  constructor(
+    private readonly tablesService: TablesService,
+    private readonly viewsService: ViewsService,
+    private readonly projectsService: ProjectsService,
+    private readonly attachmentsService: AttachmentsService,
+    private readonly columnsService: ColumnsService,
+    private readonly filtersService: FiltersService,
+    private readonly formColumnsService: FormColumnsService,
+    private readonly formsService: FormsService,
+    private readonly galleriesService: GalleriesService,
+    private readonly gridsService: GridsService,
+    private readonly projectUsersService: ProjectUsersService,
+    private readonly viewColumnsService: ViewColumnsService,
+    private readonly sortsService: SortsService,
+    private readonly bulkDataAliasService: BulkDataAliasService,
+    private readonly jobsEventService: JobsEventService,
+  ) {}
+
+  @Process('at-import')
+  async job(job: Job) {
+    const syncDB = job.data;
+
+    const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent');
+    await sMapEM.init();
+    const userRole = syncDB.user.roles
+      .split(',')
+      .reduce((rolesObj, role) => ({ [role]: true, ...rolesObj }), {});
+
+    const sMap = {
+      // static mapping records between aTblId && ncId
+      async addToMappingTbl(aTblId, ncId, ncName, ncParent?) {
+        await sMapEM.addRow({ aTblId, ncId, ncName, ncParent });
+      },
+
+      // get NcID from airtable ID
+      async getNcIdFromAtId(aId) {
+        return (await sMapEM.getRow('aTblId', aId, ['ncId']))?.ncId;
+      },
+
+      // get nc Parent from airtable ID
+      async getNcParentFromAtId(aId) {
+        return (await sMapEM.getRow('aTblId', aId, ['ncParent']))?.ncParent;
+      },
+
+      // get nc-title from airtable ID
+      async getNcNameFromAtId(aId) {
+        return (await sMapEM.getRow('aTblId', aId, ['ncName']))?.ncName;
+      },
+    };
+
+    const logBasic = (log) => {
+      this.jobsEventService.sendLog(job, { message: log });
+    };
+
+    const logDetailed = (log) => {
+      if (debugMode) this.jobsEventService.sendLog(job, { message: log });
+    };
+
+    const perfStats = [];
+
+    const recordPerfStart = () => {
+      if (!debugMode) return 0;
+      return Date.now();
+    };
+
+    const recordPerfStats = (start, event) => {
+      if (!debugMode) return;
+      const duration = Date.now() - start;
+      perfStats.push({ d: duration, e: event });
+    };
+
+    let base, baseId;
+    const start = Date.now();
+    const enableErrorLogs = false;
+    const generate_migrationStats = true;
+    const debugMode = false;
+    let g_aTblSchema = [];
+    let ncCreatedProjectSchema: any = {};
+    const ncLinkMappingTable: any[] = [];
+    const nestedLookupTbl: any[] = [];
+    const nestedRollupTbl: any[] = [];
+    const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' };
+    const storeLinks = false;
+    const ncLinkDataStore: any = {};
+    const insertedAssocRef: any = {};
+
+    const atNcAliasRef: {
+      [ncTableId: string]: {
+        [ncTitle: string]: string;
+      };
+    } = {};
+
+    const uniqueTableNameGen = getUniqueNameGenerator('sheet');
+
+    // run time counter (statistics)
+    const rtc = {
+      sort: 0,
+      filter: 0,
+      view: {
+        total: 0,
+        grid: 0,
+        gallery: 0,
+        form: 0,
+      },
+      fetchAt: {
+        count: 0,
+        time: 0,
+      },
+      migrationSkipLog: {
+        count: 0,
+        log: [],
+      },
+      data: {
+        records: 0,
+        nestedLinks: 0,
+      },
+    };
+
+    const updateMigrationSkipLog = (tbl, col, type, reason?) => {
+      rtc.migrationSkipLog.count++;
+      rtc.migrationSkipLog.log.push(
+        `tn[${tbl}] cn[${col}] type[${type}] :: ${reason}`,
+      );
+    };
+
+    // mapping table
+    //
+
+    const getAirtableSchema = async (sDB) => {
+      const start = Date.now();
+
+      if (!sDB.shareId)
+        throw {
+          message:
+            'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+        };
+
+      if (sDB.shareId.startsWith('exp')) {
+        const template = await FetchAT.readTemplate(sDB.shareId);
+        await FetchAT.initialize(template.template.exploreApplication.shareId);
+      } else {
+        await FetchAT.initialize(sDB.shareId);
+      }
+      const ft = await FetchAT.read();
+      const duration = Date.now() - start;
+      rtc.fetchAt.count++;
+      rtc.fetchAt.time += duration;
+
+      if (!ft.baseId) {
+        throw {
+          message:
+            'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+        };
+      }
+
+      const file = ft.schema;
+      baseId = ft.baseId;
+      base = new Airtable({ apiKey: sDB.apiKey }).base(baseId);
+      // store copy of airtable schema globally
+      g_aTblSchema = file.tableSchemas;
+
+      if (debugMode)
+        await writeJsonFileAsync('aTblSchema.json', ft, { spaces: 2 });
+
+      return file;
+    };
+
+    const getViewData = async (viewId) => {
+      const start = Date.now();
+      const ft = await FetchAT.readView(viewId);
+      const duration = Date.now() - start;
+      rtc.fetchAt.count++;
+      rtc.fetchAt.time += duration;
+
+      if (debugMode)
+        await writeJsonFileAsync(`${viewId}.json`, ft, { spaces: 2 });
+      return ft.view;
+    };
+
+    const getRootDbType = () => {
+      return ncCreatedProjectSchema?.bases.find((el) => el.id === syncDB.baseId)
+        ?.type;
+    };
+
+    // base mapping table
+    const aTblNcTypeMap = {
+      foreignKey: UITypes.LinkToAnotherRecord,
+      text: UITypes.SingleLineText,
+      multilineText: UITypes.LongText,
+      richText: UITypes.LongText,
+      multipleAttachment: UITypes.Attachment,
+      checkbox: UITypes.Checkbox,
+      multiSelect: UITypes.MultiSelect,
+      select: UITypes.SingleSelect,
+      collaborator: UITypes.Collaborator,
+      multiCollaborator: UITypes.Collaborator,
+      date: UITypes.Date,
+      phone: UITypes.PhoneNumber,
+      number: UITypes.Decimal,
+      rating: UITypes.Rating,
+      formula: UITypes.Formula,
+      rollup: UITypes.Rollup,
+      count: UITypes.Count,
+      lookup: UITypes.Lookup,
+      autoNumber: UITypes.AutoNumber,
+      barcode: UITypes.SingleLineText,
+      button: UITypes.Button,
+    };
+
+    //-----------------------------------------------------------------------------
+    // aTbl helper routines
+    //
+
+    const nc_sanitizeName = (name) => {
+      // replace all special characters by _
+      return name.replace(/\W+/g, '_').trim();
+    };
+
+    const nc_getSanitizedColumnName = (table, name) => {
+      let col_name = nc_sanitizeName(name);
+
+      // truncate to 60 chars if character if exceeds above 60
+      col_name = col_name?.slice(0, 60);
+
+      // for knex, replace . with _
+      const col_alias = name.trim().replace(/\./g, '_');
+
+      // check if already a column exists with same name?
+      const duplicateTitle = table.columns.find(
+        (x) => x.title?.toLowerCase() === col_alias?.toLowerCase(),
+      );
+      const duplicateColumn = table.columns.find(
+        (x) => x.column_name?.toLowerCase() === col_name?.toLowerCase(),
+      );
+      if (duplicateTitle) {
+        if (enableErrorLogs) console.log(`## Duplicate title ${col_alias}`);
+      }
+
+      return {
+        // kludge: error observed in Nc with space around column-name
+        title: col_alias + (duplicateTitle ? '_2' : ''),
+        column_name: col_name + (duplicateColumn ? '_2' : ''),
+      };
+    };
+
+    const ncSchema = {
+      tables: [],
+      tablesById: {},
+    };
+
+    // aTbl: retrieve column name from column ID
+    //
+    const aTbl_getColumnName = (colId): any => {
+      for (let i = 0; i < g_aTblSchema.length; i++) {
+        const sheetObj = g_aTblSchema[i];
+        const column = sheetObj.columns.find((col) => col.id === colId);
+        if (column !== undefined)
+          return {
+            tn: sheetObj.name,
+            cn: column.name,
+          };
+      }
+    };
+
+    const nc_DumpTableSchema = async () => {
+      console.log('[');
+      // const ncTblList = await api.base.tableList(
+      //   ncCreatedProjectSchema.id,
+      //   syncDB.baseId
+      // );
+
+      const ncTblList = { list: [] };
+      ncTblList['list'] = await this.tablesService.getAccessibleTables({
+        projectId: ncCreatedProjectSchema.id,
+        baseId: syncDB.baseId,
+        roles: userRole,
+      });
+
+      for (let i = 0; i < ncTblList.list.length; i++) {
+        // const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
+        const ncTbl = await this.tablesService.getTableWithAccessibleViews({
+          tableId: ncTblList.list[i].id,
+          user: syncDB.user,
+        });
+        console.log(JSON.stringify(ncTbl, null, 2));
+        console.log(',');
+      }
+      console.log(']');
+    };
+
+    // retrieve nc column schema from using aTbl field ID as reference
+    //
+    const nc_getColumnSchema = async (aTblFieldId) => {
+      const ncTblId = await sMap.getNcParentFromAtId(aTblFieldId);
+      const ncColId = await sMap.getNcIdFromAtId(aTblFieldId);
+
+      // not migrated column, skip
+      if (ncColId === undefined || ncTblId === undefined) return 0;
+
+      return ncSchema.tablesById[ncTblId].columns.find((x) => x.id === ncColId);
+    };
+
+    // retrieve nc table schema using table name
+    // optimize: create a look-up table & re-use information
+    //
+    const nc_getTableSchema = async (tableName) => {
+      return ncSchema.tables.find((x) => x.title === tableName);
+    };
+
+    // delete project if already exists
+    const init = async ({
+      projectName,
+    }: {
+      projectName?: string;
+      projectId?: string;
+    }) => {
+      // delete 'sample' project if already exists
+      const x = { list: [] };
+      x['list'] = await this.projectsService.projectList({
+        user: { id: syncDB.user.id, roles: extractRolesObj(syncDB.user.roles) },
+      });
+
+      const sampleProj = x.list.find((a) => a.title === projectName);
+      if (sampleProj) {
+        await this.projectsService.projectSoftDelete({
+          projectId: sampleProj.id,
+        });
+      }
+      logDetailed('Init');
+    };
+
+    // map UIDT
+    //
+    const getNocoType = (col) => {
+      // start with default map
+      let ncType = aTblNcTypeMap[col.type];
+
+      // types email & url are marked as text
+      // types currency & percent, duration are marked as number
+      // types createTime & modifiedTime are marked as formula
+
+      switch (col.type) {
+        case 'text':
+          if (col.typeOptions?.validatorName === 'email')
+            ncType = UITypes.Email;
+          else if (col.typeOptions?.validatorName === 'url')
+            ncType = UITypes.URL;
+          break;
+
+        case 'number':
+          // kludge: currency validation error with decimal places
+          if (col.typeOptions?.format === 'percentV2') ncType = UITypes.Percent;
+          else if (col.typeOptions?.format === 'duration')
+            ncType = UITypes.Duration;
+          else if (col.typeOptions?.format === 'currency')
+            ncType = UITypes.Currency;
+          else if (col.typeOptions?.precision > 0) ncType = UITypes.Decimal;
+          break;
+
+        case 'formula':
+          if (col.typeOptions?.formulaTextParsed === 'CREATED_TIME()')
+            ncType = UITypes.DateTime;
+          else if (
+            col.typeOptions?.formulaTextParsed === 'LAST_MODIFIED_TIME()'
+          )
+            ncType = UITypes.DateTime;
+          break;
+
+        case 'computation':
+          if (col.typeOptions?.resultType === 'collaborator')
+            ncType = UITypes.Collaborator;
+          break;
+
+        case 'date':
+          if (col.typeOptions?.isDateTime) ncType = UITypes.DateTime;
+          break;
+      }
+
+      return ncType;
+    };
+
+    // retrieve additional options associated with selected data types
+    //
+    const getNocoTypeOptions = async (col: any): Promise<any> => {
+      switch (col.type) {
+        case 'select':
+        case 'multiSelect': {
+          // prepare options list in CSV format
+          // note: NC doesn't allow comma's in options
+          //
+          const options = [];
+          let order = 1;
+          for (const [, value] of Object.entries(col.typeOptions.choices)) {
+            // replace commas with dot for multiselect
+            if (col.type === 'multiSelect') {
+              (value as any).name = (value as any).name.replace(/,/g, '.');
+            }
+            // we don't allow empty records, placeholder instead
+            if ((value as any).name === '') {
+              (value as any).name = 'nc_empty';
+            }
+            // enumerate duplicates (we don't allow them)
+            // TODO fix record mapping (this causes every record to map first option,
+            //  we can't handle them using data api as they don't provide option id
+            //  within data we might instead get the correct mapping from schema file )
+            let dupNo = 1;
+            const defaultName = (value as any).name;
+            while (
+              options.find(
+                (el) =>
+                  el.title.toLowerCase() === (value as any).name.toLowerCase(),
+              )
+            ) {
+              (value as any).name = `${defaultName}_${dupNo++}`;
+            }
+            options.push({
+              order: order++,
+              title: (value as any).name,
+              color: selectColors[(value as any).color]
+                ? selectColors[(value as any).color]
+                : tinycolor.random().toHexString(),
+            });
+
+            await sMap.addToMappingTbl(
+              (value as any).id,
+              undefined,
+              (value as any).name,
+            );
+          }
+          return { type: col.type, data: options };
+        }
+        default:
+          return { type: undefined };
+      }
+    };
+
+    // convert to Nc schema (basic, excluding relations)
+    //
+    const tablesPrepare = async (tblSchema: any[]) => {
+      const tables: any[] = [];
+
+      for (let i = 0; i < tblSchema.length; ++i) {
+        const table: any = {};
+
+        if (syncDB.options.syncViews) {
+          rtc.view.total += tblSchema[i].views.reduce(
+            (acc, cur) =>
+              ['grid', 'form', 'gallery'].includes(cur.type) ? ++acc : acc,
+            0,
+          );
+        } else {
+          rtc.view.total = tblSchema.length;
+        }
+
+        // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id;
+        table.title = tblSchema[i].name;
+        let sanitizedName = nc_sanitizeName(tblSchema[i].name);
+
+        // truncate to 50 chars if character if exceeds above 50
+        // upto 64 should be fine but we are keeping it to 50 since
+        // meta project adds prefix as well
+        sanitizedName = sanitizedName?.slice(0, 50);
+
+        // check for duplicate and populate a unique name if already exist
+        table.table_name = uniqueTableNameGen(sanitizedName);
+
+        const uniqueColNameGen = getUniqueNameGenerator('field');
+        table.columns = [];
+        const sysColumns = [
+          {
+            title: ncSysFields.id,
+            column_name: ncSysFields.id,
+            uidt: UITypes.ID,
+            meta: {
+              ag: 'nc',
+            },
+          },
+          {
+            title: ncSysFields.hash,
+            column_name: ncSysFields.hash,
+            uidt: UITypes.SingleLineText,
+            system: true,
+          },
+        ];
+
+        for (let j = 0; j < tblSchema[i].columns.length; j++) {
+          const col = tblSchema[i].columns[j];
+
+          // skip link, lookup, rollup fields in this iteration
+          if (['foreignKey', 'lookup', 'rollup'].includes(col.type)) {
+            continue;
+          }
+
+          // base column schema
+          const ncName: any = nc_getSanitizedColumnName(table, col.name);
+          const ncCol: any = {
+            // Enable to use aTbl identifiers as is: id: col.id,
+            title: ncName.title,
+            column_name: uniqueColNameGen(ncName.column_name),
+            uidt: getNocoType(col),
+          };
+
+          // not supported datatype: pure formula field
+          // allow formula based computed fields (created time/ modified time to go through)
+          if (ncCol.uidt === UITypes.Formula) {
+            updateMigrationSkipLog(
+              tblSchema[i].name,
+              ncName.title,
+              col.type,
+              'column type not supported',
+            );
+            continue;
+          }
+
+          // change from default 'tinytext' as airtable allows more than 255 characters
+          // for single line text column type
+          if (col.type === 'text') ncCol.dt = 'text';
+
+          // #fix-2363-decimal-out-of-range
+          if (['sqlite3', 'mysql2'].includes(getRootDbType())) {
+            if (ncCol.uidt === UITypes.Decimal) {
+              ncCol.dt = 'double';
+              ncCol.dtxp = 22;
+              ncCol.dtxs = '2';
+            }
+          }
+
+          // additional column parameters when applicable
+          const colOptions = await getNocoTypeOptions(col);
+
+          switch (colOptions.type) {
+            case 'select':
+            case 'multiSelect':
+              ncCol.colOptions = {
+                options: [...colOptions.data],
+              };
+
+              if (['mysql', 'mysql2'].includes(getRootDbType())) {
+                // if options are empty, configure '' as an option
+                ncCol.dtxp =
+                  colOptions.data
+                    .map((el) => `'${el.title.replace(/'/gi, "''")}'`)
+                    .join(',') || "''";
+              }
+
+              break;
+            case undefined:
+              break;
+          }
+          table.columns.push(ncCol);
+        }
+        table.columns.push(sysColumns[0]);
+        table.columns.push(sysColumns[1]);
+
+        tables.push(table);
+      }
+      return tables;
+    };
+
+    const nocoCreateBaseSchema = async (aTblSchema) => {
+      // base schema preparation: exclude
+      const tables: any[] = await tablesPrepare(aTblSchema);
+
+      // for each table schema, create nc table
+      for (let idx = 0; idx < tables.length; idx++) {
+        logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`);
+
+        logDetailed(`NC API: base.tableCreate ${tables[idx].title}`);
+
+        let _perfStart = recordPerfStart();
+        const table = await this.tablesService.tableCreate({
+          baseId: syncDB.baseId,
+          projectId: ncCreatedProjectSchema.id,
+          table: tables[idx],
+          user: syncDB.user,
+        });
+        recordPerfStats(_perfStart, 'dbTable.create');
+
+        updateNcTblSchema(table);
+
+        // update mapping table
+        await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title);
+        for (let colIdx = 0; colIdx < table.columns.length; colIdx++) {
+          const aId = aTblSchema[idx].columns.find(
+            (x) =>
+              x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title,
+          )?.id;
+          if (aId)
+            await sMap.addToMappingTbl(
+              aId,
+              table.columns[colIdx].id,
+              table.columns[colIdx].title,
+              table.id,
+            );
+        }
+
+        // update default view name- to match it to airtable view name
+        logDetailed(`NC API: dbView.list ${table.id}`);
+        _perfStart = recordPerfStart();
+        const view = { list: [] };
+        view['list'] = await this.viewsService.viewList({
+          tableId: table.id,
+          user: { roles: userRole },
+        });
+        recordPerfStats(_perfStart, 'dbView.list');
+
+        const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid');
+        logDetailed(
+          `NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`,
+        );
+        _perfStart = recordPerfStart();
+        await this.viewsService.viewUpdate({
+          viewId: view.list[0].id,
+          view: { title: aTbl_grid.name },
+        });
+        recordPerfStats(_perfStart, 'dbView.update');
+
+        await updateNcTblSchemaById(table.id);
+
+        await sMap.addToMappingTbl(
+          aTbl_grid.id,
+          table.views[0].id,
+          aTbl_grid.name,
+          table.id,
+        );
+      }
+
+      return tables;
+    };
+
+    const nocoCreateLinkToAnotherRecord = async (aTblSchema) => {
+      // Link to another RECORD
+      for (let idx = 0; idx < aTblSchema.length; idx++) {
+        const aTblLinkColumns = aTblSchema[idx].columns.filter(
+          (x) => x.type === 'foreignKey',
+        );
+
+        // Link columns exist
+        //
+        if (aTblLinkColumns.length) {
+          for (let i = 0; i < aTblLinkColumns.length; i++) {
+            logDetailed(
+              `[${idx + 1}/${aTblSchema.length}] Configuring Links :: [${
+                i + 1
+              }/${aTblLinkColumns.length}] ${aTblSchema[idx].name}`,
+            );
+
+            // for self links, there is no symmetric column
+            {
+              const src = aTbl_getColumnName(aTblLinkColumns[i].id);
+              const dst = aTbl_getColumnName(
+                aTblLinkColumns[i].typeOptions?.symmetricColumnId,
+              );
+              logDetailed(
+                `LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst?.tn}:${dst?.cn}`,
+              );
+            }
+
+            // check if link already established?
+            if (!nc_isLinkExists(aTblLinkColumns[i].id)) {
+              // parent table ID
+              const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+
+              // find child table name from symmetric column ID specified
+              // self link, symmetricColumnId field will be undefined
+              const childTable = aTbl_getColumnName(
+                aTblLinkColumns[i].typeOptions?.symmetricColumnId,
+              );
+
+              // retrieve child table ID (nc) from table name
+              let childTableId = srcTableId;
+              if (childTable) {
+                childTableId = (await nc_getTableSchema(childTable.tn)).id;
+              }
+
+              // check if already a column exists with this name?
+              let _perfStart = recordPerfStart();
+              const srcTbl: any =
+                await this.tablesService.getTableWithAccessibleViews({
+                  tableId: srcTableId,
+                  user: syncDB.user,
+                });
+              recordPerfStats(_perfStart, 'dbTable.read');
+
+              // create link
+              const ncName = nc_getSanitizedColumnName(
+                srcTbl,
+                aTblLinkColumns[i].name,
+              );
+
+              // LTAR alias ref to AT
+              atNcAliasRef[srcTbl.id] = atNcAliasRef[srcTbl.id] || {};
+              atNcAliasRef[srcTbl.id][ncName.title] = aTblLinkColumns[i].name;
+
+              logDetailed(
+                `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}`,
+              );
+              _perfStart = recordPerfStart();
+              const ncTbl: any = await this.columnsService.columnAdd({
+                tableId: srcTableId,
+                column: {
+                  uidt: UITypes.LinkToAnotherRecord,
+                  title: ncName.title,
+                  column_name: ncName.column_name,
+                  parentId: srcTableId,
+                  childId: childTableId,
+                  type: 'mm',
+                },
+                req: {
+                  user: syncDB.user.email,
+                  clientIp: '',
+                },
+              });
+              recordPerfStats(_perfStart, 'dbTableColumn.create');
+
+              updateNcTblSchema(ncTbl);
+
+              const ncId = ncTbl.columns.find(
+                (x) => x.title === ncName.title,
+              )?.id;
+              await sMap.addToMappingTbl(
+                aTblLinkColumns[i].id,
+                ncId,
+                ncName.title,
+                ncTbl.id,
+              );
+
+              // store link information in separate table
+              // this information will be helpful in identifying relation pair
+              const link = {
+                nc: {
+                  title: ncName.title,
+                  parentId: srcTableId,
+                  childId: childTableId,
+                  type: 'mm',
+                },
+                aTbl: {
+                  tblId: aTblSchema[idx].id,
+                  ...aTblLinkColumns[i],
+                },
+              };
+
+              ncLinkMappingTable.push(link);
+            } else {
+              // if link already exists, we need to change name of linked column
+              // to what is represented in airtable
+
+              // 1. extract associated link information from link table
+              // 2. retrieve parent table information (source)
+              // 3. using foreign parent & child column ID, find associated mapping in child table
+              // 4. update column name
+              const x = ncLinkMappingTable.findIndex(
+                (x) =>
+                  x.aTbl.tblId ===
+                    aTblLinkColumns[i].typeOptions.foreignTableId &&
+                  x.aTbl.id ===
+                    aTblLinkColumns[i].typeOptions.symmetricColumnId,
+              );
+
+              let _perfStart = recordPerfStart();
+              const childTblSchema: any =
+                await this.tablesService.getTableWithAccessibleViews({
+                  tableId: ncLinkMappingTable[x].nc.childId,
+                  user: syncDB.user,
+                });
+              recordPerfStats(_perfStart, 'dbTable.read');
+
+              _perfStart = recordPerfStart();
+              const parentTblSchema: any =
+                await this.tablesService.getTableWithAccessibleViews({
+                  tableId: ncLinkMappingTable[x].nc.parentId,
+                  user: syncDB.user,
+                });
+              recordPerfStats(_perfStart, 'dbTable.read');
+
+              let parentLinkColumn = parentTblSchema.columns.find(
+                (col) => col.title === ncLinkMappingTable[x].nc.title,
+              );
+
+              if (parentLinkColumn === undefined) {
+                updateMigrationSkipLog(
+                  parentTblSchema?.title,
+                  ncLinkMappingTable[x].nc.title,
+                  UITypes.LinkToAnotherRecord,
+                  'Link error',
+                );
+                continue;
+              }
+
+              // hack // fix me
+              if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') {
+                parentLinkColumn = parentTblSchema.columns.find(
+                  (col) => col.title === ncLinkMappingTable[x].nc.title + '_2',
+                );
+              }
+
+              let childLinkColumn: any = {};
+
+              if (parentLinkColumn.colOptions.type == 'hm') {
+                // for hm:
+                // mapping between child & parent column id is direct
+                //
+                childLinkColumn = childTblSchema.columns.find(
+                  (col) =>
+                    col.uidt === UITypes.LinkToAnotherRecord &&
+                    col.colOptions.fk_child_column_id ===
+                      parentLinkColumn.colOptions.fk_child_column_id &&
+                    col.colOptions.fk_parent_column_id ===
+                      parentLinkColumn.colOptions.fk_parent_column_id,
+                );
+              } else {
+                // for mm:
+                // mapping between child & parent column id is inverted
+                //
+                childLinkColumn = childTblSchema.columns.find(
+                  (col) =>
+                    col.uidt === UITypes.LinkToAnotherRecord &&
+                    col.colOptions.fk_child_column_id ===
+                      parentLinkColumn.colOptions.fk_parent_column_id &&
+                    col.colOptions.fk_parent_column_id ===
+                      parentLinkColumn.colOptions.fk_child_column_id &&
+                    col.colOptions.fk_mm_model_id ===
+                      parentLinkColumn.colOptions.fk_mm_model_id,
+                );
+              }
+
+              // check if already a column exists with this name?
+              const duplicate = childTblSchema.columns.find(
+                (x) => x.title === aTblLinkColumns[i].name,
+              );
+              const suffix = duplicate ? '_2' : '';
+              if (duplicate)
+                if (enableErrorLogs)
+                  console.log(`## Duplicate ${aTblLinkColumns[i].name}`);
+
+              // rename
+              // note that: current rename API requires us to send all parameters,
+              // not just title being renamed
+              const ncName = nc_getSanitizedColumnName(
+                childTblSchema,
+                aTblLinkColumns[i].name,
+              );
+
+              logDetailed(
+                `NC API: dbTableColumn.update rename symmetric column ${ncName.title}`,
+              );
+              _perfStart = recordPerfStart();
+              const ncTbl: any = await this.columnsService.columnUpdate({
+                columnId: childLinkColumn.id,
+                column: {
+                  ...childLinkColumn,
+                  title: ncName.title,
+                  column_name: ncName.column_name,
+                },
+              });
+              recordPerfStats(_perfStart, 'dbTableColumn.update');
+
+              updateNcTblSchema(ncTbl);
+
+              const ncId = ncTbl.columns.find(
+                (x) => x.title === aTblLinkColumns[i].name + suffix,
+              )?.id;
+              await sMap.addToMappingTbl(
+                aTblLinkColumns[i].id,
+                ncId,
+                aTblLinkColumns[i].name + suffix,
+                ncTbl.id,
+              );
+            }
+          }
+        }
+      }
+    };
+
+    const nocoCreateLookups = async (aTblSchema) => {
+      // LookUps
+      for (let idx = 0; idx < aTblSchema.length; idx++) {
+        const aTblColumns = aTblSchema[idx].columns.filter(
+          (x) => x.type === 'lookup',
+        );
+
+        // parent table ID
+        const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+        const srcTableSchema = ncSchema.tablesById[srcTableId];
+
+        if (aTblColumns.length) {
+          // Lookup
+          for (let i = 0; i < aTblColumns.length; i++) {
+            logDetailed(
+              `[${idx + 1}/${aTblSchema.length}] Configuring Lookup :: [${
+                i + 1
+              }/${aTblColumns.length}] ${aTblSchema[idx].name}`,
+            );
+
+            // something is not right, skip
+            if (
+              aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds
+                ?.length
+            ) {
+              if (enableErrorLogs)
+                console.log(`## Invalid column IDs mapped; skip`);
+
+              updateMigrationSkipLog(
+                srcTableSchema.title,
+                aTblColumns[i].name,
+                aTblColumns[i].type,
+                'invalid column ID in dependency list',
+              );
+              continue;
+            }
+
+            const ncRelationColumnId = await sMap.getNcIdFromAtId(
+              aTblColumns[i].typeOptions.relationColumnId,
+            );
+            const ncLookupColumnId = await sMap.getNcIdFromAtId(
+              aTblColumns[i].typeOptions.foreignTableRollupColumnId,
+            );
+
+            if (
+              ncLookupColumnId === undefined ||
+              ncRelationColumnId === undefined
+            ) {
+              aTblColumns[i]['srcTableId'] = srcTableId;
+              nestedLookupTbl.push(aTblColumns[i]);
+              continue;
+            }
+
+            const ncName = nc_getSanitizedColumnName(
+              srcTableSchema,
+              aTblColumns[i].name,
+            );
+
+            logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
+            const _perfStart = recordPerfStart();
+            const ncTbl: any = await this.columnsService.columnAdd({
+              tableId: srcTableId,
+              column: {
+                uidt: UITypes.Lookup,
+                title: ncName.title,
+                column_name: ncName.column_name,
+                fk_relation_column_id: ncRelationColumnId,
+                fk_lookup_column_id: ncLookupColumnId,
+              },
+              req: {
+                user: syncDB.user.email,
+                clientIp: '',
+              },
+            });
+            recordPerfStats(_perfStart, 'dbTableColumn.create');
+
+            updateNcTblSchema(ncTbl);
+
+            const ncId = ncTbl.columns.find(
+              (x) => x.title === aTblColumns[i].name,
+            )?.id;
+            await sMap.addToMappingTbl(
+              aTblColumns[i].id,
+              ncId,
+              aTblColumns[i].name,
+              ncTbl.id,
+            );
+          }
+        }
+      }
+
+      let level = 2;
+      let nestedCnt = 0;
+      while (nestedLookupTbl.length) {
+        // if nothing has changed from previous iteration, skip rest
+        if (nestedCnt === nestedLookupTbl.length) {
+          for (let i = 0; i < nestedLookupTbl.length; i++) {
+            const fTblField =
+              nestedLookupTbl[i].typeOptions.foreignTableRollupColumnId;
+            const name = aTbl_getColumnName(fTblField);
+            updateMigrationSkipLog(
+              ncSchema.tablesById[nestedLookupTbl[i].srcTableId]?.title,
+              nestedLookupTbl[i].name,
+              nestedLookupTbl[i].type,
+              `foreign table field not found [${name.tn}/${name.cn}]`,
+            );
+          }
+          if (enableErrorLogs)
+            console.log(
+              `## Failed to configure ${nestedLookupTbl.length} lookups`,
+            );
+          break;
+        }
+
+        // Nested lookup
+        nestedCnt = nestedLookupTbl.length;
+        for (let i = 0; i < nestedLookupTbl.length; i++) {
+          const srcTableId = nestedLookupTbl[0].srcTableId;
+          const srcTableSchema = ncSchema.tablesById[srcTableId];
+
+          const ncRelationColumnId = await sMap.getNcIdFromAtId(
+            nestedLookupTbl[0].typeOptions.relationColumnId,
+          );
+          const ncLookupColumnId = await sMap.getNcIdFromAtId(
+            nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId,
+          );
+
+          if (
+            ncLookupColumnId === undefined ||
+            ncRelationColumnId === undefined
+          ) {
+            continue;
+          }
+
+          const ncName = nc_getSanitizedColumnName(
+            srcTableSchema,
+            nestedLookupTbl[0].name,
+          );
+
+          logDetailed(
+            `Configuring Nested Lookup: Level-${level} [${i + 1}/${nestedCnt} ${
+              ncName.title
+            }]`,
+          );
+
+          logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
+          const _perfStart = recordPerfStart();
+          const ncTbl: any = await this.columnsService.columnAdd({
+            tableId: srcTableId,
+            column: {
+              uidt: UITypes.Lookup,
+              title: ncName.title,
+              column_name: ncName.column_name,
+              fk_relation_column_id: ncRelationColumnId,
+              fk_lookup_column_id: ncLookupColumnId,
+            },
+            req: {
+              user: syncDB.user.email,
+              clientIp: '',
+            },
+          });
+          recordPerfStats(_perfStart, 'dbTableColumn.create');
+
+          updateNcTblSchema(ncTbl);
+
+          const ncId = ncTbl.columns.find(
+            (x) => x.title === nestedLookupTbl[0].name,
+          )?.id;
+          await sMap.addToMappingTbl(
+            nestedLookupTbl[0].id,
+            ncId,
+            nestedLookupTbl[0].name,
+            ncTbl.id,
+          );
+
+          // remove entry
+          nestedLookupTbl.splice(0, 1);
+        }
+        level++;
+      }
+    };
+
+    const getRollupNcFunction = (aTblFunction) => {
+      const fn = aTblFunction.split('(')[0];
+      const aTbl_ncRollUp = {
+        AND: '',
+        ARRAYCOMPACT: '',
+        ARRAYJOIN: '',
+        ARRAYUNIQUE: '',
+        AVERAGE: 'average',
+        CONCATENATE: '',
+        COUNT: 'count',
+        COUNTA: '',
+        COUNTALL: '',
+        MAX: 'max',
+        MIN: 'min',
+        OR: '',
+        SUM: 'sum',
+        XOR: '',
+      };
+      return aTbl_ncRollUp[fn];
+    };
+
+    const nocoCreateRollup = async (aTblSchema) => {
+      // Rollup
+      for (let idx = 0; idx < aTblSchema.length; idx++) {
+        const aTblColumns = aTblSchema[idx].columns.filter(
+          (x) => x.type === 'rollup',
+        );
+
+        // parent table ID
+        const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+        const srcTableSchema = ncSchema.tablesById[srcTableId];
+
+        if (aTblColumns.length) {
+          // rollup exist
+          for (let i = 0; i < aTblColumns.length; i++) {
+            logDetailed(
+              `[${idx + 1}/${aTblSchema.length}] Configuring Rollup :: [${
+                i + 1
+              }/${aTblColumns.length}] ${aTblSchema[idx].name}`,
+            );
+
+            // fetch associated rollup function
+            // skip column creation if supported rollup function does not exist
+            const ncRollupFn = getRollupNcFunction(
+              aTblColumns[i].typeOptions.formulaTextParsed,
+            );
+
+            if (ncRollupFn === '' || ncRollupFn === undefined) {
+              updateMigrationSkipLog(
+                srcTableSchema.title,
+                aTblColumns[i].name,
+                aTblColumns[i].type,
+                `rollup function ${aTblColumns[i].typeOptions.formulaTextParsed} not supported`,
+              );
+              continue;
+            }
+
+            // something is not right, skip
+            if (
+              aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds
+                ?.length
+            ) {
+              if (enableErrorLogs)
+                console.log(`## Invalid column IDs mapped; skip`);
+
+              updateMigrationSkipLog(
+                srcTableSchema.title,
+                aTblColumns[i].name,
+                aTblColumns[i].type,
+                'invalid column ID in dependency list',
+              );
+              continue;
+            }
+
+            const ncRelationColumnId = await sMap.getNcIdFromAtId(
+              aTblColumns[i].typeOptions.relationColumnId,
+            );
+            const ncRollupColumnId = await sMap.getNcIdFromAtId(
+              aTblColumns[i].typeOptions.foreignTableRollupColumnId,
+            );
+
+            if (ncRollupColumnId === undefined) {
+              aTblColumns[i]['srcTableId'] = srcTableId;
+              nestedRollupTbl.push(aTblColumns[i]);
+              continue;
+            }
+
+            // skip, if rollup column was pointing to another virtual column
+            const ncColSchema = await nc_getColumnSchema(
+              aTblColumns[i].typeOptions.foreignTableRollupColumnId,
+            );
+            if (
+              ncColSchema?.uidt === UITypes.Formula ||
+              ncColSchema?.uidt === UITypes.Lookup ||
+              ncColSchema?.uidt === UITypes.Rollup ||
+              ncColSchema?.uidt === UITypes.Checkbox
+            ) {
+              updateMigrationSkipLog(
+                srcTableSchema.title,
+                aTblColumns[i].name,
+                aTblColumns[i].type,
+                'rollup referring to a column type not supported currently',
+              );
+              continue;
+            }
+
+            const ncName = nc_getSanitizedColumnName(
+              srcTableSchema,
+              aTblColumns[i].name,
+            );
+
+            logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`);
+            const _perfStart = recordPerfStart();
+            const ncTbl: any = await this.columnsService.columnAdd({
+              tableId: srcTableId,
+              column: {
+                uidt: UITypes.Rollup,
+                title: ncName.title,
+                column_name: ncName.column_name,
+                fk_relation_column_id: ncRelationColumnId,
+                fk_rollup_column_id: ncRollupColumnId,
+                rollup_function: ncRollupFn,
+              },
+              req: {
+                user: syncDB.user.email,
+                clientIp: '',
+              },
+            });
+            recordPerfStats(_perfStart, 'dbTableColumn.create');
+
+            updateNcTblSchema(ncTbl);
+
+            const ncId = ncTbl.columns.find(
+              (x) => x.title === aTblColumns[i].name,
+            )?.id;
+            await sMap.addToMappingTbl(
+              aTblColumns[i].id,
+              ncId,
+              aTblColumns[i].name,
+              ncTbl.id,
+            );
+          }
+        }
+      }
+      logDetailed(`Nested rollup: ${nestedRollupTbl.length}`);
+    };
+
+    const nocoLookupForRollup = async () => {
+      const nestedCnt = nestedLookupTbl.length;
+      for (let i = 0; i < nestedLookupTbl.length; i++) {
+        const srcTableId = nestedLookupTbl[0].srcTableId;
+        const srcTableSchema = ncSchema.tablesById[srcTableId];
+
+        const ncRelationColumnId = await sMap.getNcIdFromAtId(
+          nestedLookupTbl[0].typeOptions.relationColumnId,
+        );
+        const ncLookupColumnId = await sMap.getNcIdFromAtId(
+          nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId,
+        );
+
+        if (
+          ncLookupColumnId === undefined ||
+          ncRelationColumnId === undefined
+        ) {
+          continue;
+        }
+
+        const ncName = nc_getSanitizedColumnName(
+          srcTableSchema,
+          nestedLookupTbl[0].name,
+        );
+
+        logDetailed(
+          `Configuring Lookup over Rollup :: [${i + 1}/${nestedCnt}] ${
+            ncName.title
+          }`,
+        );
+
+        logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
+        const _perfStart = recordPerfStart();
+        const ncTbl: any = await this.columnsService.columnAdd({
+          tableId: srcTableId,
+          column: {
+            uidt: UITypes.Lookup,
+            title: ncName.title,
+            column_name: ncName.column_name,
+            fk_relation_column_id: ncRelationColumnId,
+            fk_lookup_column_id: ncLookupColumnId,
+          },
+          req: {
+            user: syncDB.user.email,
+            clientIp: '',
+          },
+        });
+        recordPerfStats(_perfStart, 'dbTableColumn.create');
+
+        updateNcTblSchema(ncTbl);
+
+        const ncId = ncTbl.columns.find(
+          (x) => x.title === nestedLookupTbl[0].name,
+        )?.id;
+        await sMap.addToMappingTbl(
+          nestedLookupTbl[0].id,
+          ncId,
+          nestedLookupTbl[0].name,
+          ncTbl.id,
+        );
+
+        // remove entry
+        nestedLookupTbl.splice(0, 1);
+      }
+    };
+
+    const nocoSetPrimary = async (aTblSchema) => {
+      for (let idx = 0; idx < aTblSchema.length; idx++) {
+        logDetailed(
+          `[${idx + 1}/${aTblSchema.length}] Configuring Display value : ${
+            aTblSchema[idx].name
+          }`,
+        );
+
+        const pColId = aTblSchema[idx].primaryColumnId;
+        const ncColId = await sMap.getNcIdFromAtId(pColId);
+
+        // skip primary column configuration if we field not migrated
+        if (ncColId) {
+          logDetailed(`NC API: dbTableColumn.primaryColumnSet`);
+          const _perfStart = recordPerfStart();
+          await this.columnsService.columnSetAsPrimary({ columnId: ncColId });
+          recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet');
+
+          // update schema
+          const ncTblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+          await updateNcTblSchemaById(ncTblId);
+        }
+      }
+    };
+
+    // retrieve nc-view column ID from corresponding nc-column ID
+    const nc_getViewColumnId = async (viewId, viewType, ncColumnId) => {
+      // retrieve view Info
+      let viewDetails;
+
+      const _perfStart = recordPerfStart();
+      if (viewType === 'form') {
+        viewDetails = (
+          await this.formsService.formViewGet({ formViewId: viewId })
+        ).columns;
+        recordPerfStats(_perfStart, 'dbView.formRead');
+      } else if (viewType === 'gallery') {
+        viewDetails = (
+          await this.galleriesService.galleryViewGet({ galleryViewId: viewId })
+        ).columns;
+        recordPerfStats(_perfStart, 'dbView.galleryRead');
+      } else {
+        viewDetails = await this.viewColumnsService.columnList({
+          viewId: viewId,
+        });
+        recordPerfStats(_perfStart, 'dbView.gridColumnsList');
+      }
+
+      return viewDetails.find((x) => x.fk_column_id === ncColumnId)?.id;
+    };
+
+    //////////  Data processing
+
+    const nocoBaseDataProcessing_v2 = async (sDB, table, record) => {
+      const recordHash = hash(record);
+      const rec = { ...record.fields };
+
+      // kludge -
+      // trim spaces on either side of column name
+      // leads to error in NocoDB
+      Object.keys(rec).forEach((key) => {
+        const replacedKey = key.trim().replace(/\./g, '_');
+        if (key !== replacedKey) {
+          rec[replacedKey] = rec[key];
+          delete rec[key];
+        }
+      });
+
+      // post-processing on the record
+      for (const [key, value] of Object.entries(
+        rec as { [key: string]: any },
+      )) {
+        // retrieve datatype
+        const dt = table.columns.find((x) => x.title === key)?.uidt;
+
+        // always process LTAR, Lookup, and Rollup columns as we delete the key after processing
+        if (
+          !value &&
+          dt !== UITypes.LinkToAnotherRecord &&
+          dt !== UITypes.Lookup &&
+          dt !== UITypes.Rollup
+        ) {
+          rec[key] = null;
+          continue;
+        }
+
+        switch (dt) {
+          // https://www.npmjs.com/package/validator
+          // default value: digits_after_decimal: [2]
+          // if currency, set decimal place to 2
+          //
+          case UITypes.Currency:
+            rec[key] = (+value).toFixed(2);
+            break;
+
+          // we will pick up LTAR once all table data's are in place
+          case UITypes.LinkToAnotherRecord:
+            if (storeLinks) {
+              if (ncLinkDataStore[table.title][record.id] === undefined)
+                ncLinkDataStore[table.title][record.id] = {
+                  id: record.id,
+                  fields: {},
+                };
+              ncLinkDataStore[table.title][record.id]['fields'][key] = value;
+            }
+            delete rec[key];
+            break;
+
+          // these will be automatically populated depending on schema configuration
+          case UITypes.Lookup:
+          case UITypes.Rollup:
+            delete rec[key];
+            break;
+
+          case UITypes.Collaborator:
+            // in case of multi-collaborator, this will be an array
+            if (Array.isArray(value)) {
+              let collaborators = '';
+              for (let i = 0; i < value.length; i++) {
+                collaborators += `${value[i]?.name} <${value[i]?.email}>, `;
+                rec[key] = collaborators;
+              }
+            } else rec[key] = `${value?.name} <${value?.email}>`;
+            break;
+
+          case UITypes.Button:
+            rec[key] = `${value?.label} <${value?.url}>`;
+            break;
+
+          case UITypes.DateTime:
+          case UITypes.CreateTime:
+          case UITypes.LastModifiedTime:
+            rec[key] = dayjs(value).format('YYYY-MM-DD HH:mm');
+            break;
+
+          case UITypes.Date:
+            if (/\d{5,}/.test(value)) {
+              // skip
+              rec[key] = null;
+              logBasic(`:: Invalid date ${value}`);
+            } else {
+              rec[key] = dayjs(value).format('YYYY-MM-DD');
+            }
+            break;
+
+          case UITypes.SingleSelect:
+            if (value === '') {
+              rec[key] = 'nc_empty';
+            }
+            rec[key] = value;
+            break;
+
+          case UITypes.MultiSelect:
+            rec[key] = value
+              ?.map((v) => {
+                if (v === '') {
+                  return 'nc_empty';
+                }
+                return `${v.replace(/,/g, '.')}`;
+              })
+              .join(',');
+            break;
+
+          case UITypes.Attachment:
+            if (!syncDB.options.syncAttachment) rec[key] = null;
+            else {
+              let tempArr = [];
+
+              try {
+                logBasic(
+                  ` :: Retrieving attachment :: ${value
+                    ?.map((a) => a.filename?.split('?')?.[0])
+                    .join(', ')}`,
+                );
+                tempArr = await this.attachmentsService.uploadViaURL({
+                  path: `noco/${sDB.projectName}/${table.title}/${key}`,
+                  urls: value?.map((attachment) => ({
+                    fileName: attachment.filename?.split('?')?.[0],
+                    url: attachment.url,
+                    size: attachment.size,
+                    mimetype: attachment.type,
+                  })),
+                });
+              } catch (e) {
+                console.log(e);
+              }
+
+              rec[key] = JSON.stringify(tempArr);
+            }
+            break;
+
+          case UITypes.SingleLineText:
+            // Barcode data
+            if (value?.text) {
+              rec[key] = value.text;
+            }
+            break;
+
+          default:
+            break;
+        }
+      }
+
+      // insert airtable record ID explicitly into each records
+      rec[ncSysFields.id] = record.id;
+      rec[ncSysFields.hash] = recordHash;
+
+      return rec;
+    };
+
+    const nocoReadDataSelected = async (projName, table, callback, fields) => {
+      return new Promise((resolve, reject) => {
+        base(table.title)
+          .select({
+            pageSize: 100,
+            // maxRecords: 100,
+            fields: fields,
+          })
+          .eachPage(
+            async function page(records, fetchNextPage) {
+              // This function (`page`) will get called for each page of records.
+              // records.forEach(record => callback(table, record));
+              logBasic(
+                `:: ${table.title} / ${fields} : ${
+                  recordCnt + 1
+                } ~ ${(recordCnt += 100)}`,
+              );
+              await Promise.all(
+                records.map((r) => callback(projName, table, r, fields)),
+              );
+
+              // To fetch the next page of records, call `fetchNextPage`.
+              // If there are more records, `page` will get called again.
+              // If there are no more records, `done` will get called.
+              fetchNextPage();
+            },
+            function done(err) {
+              if (err) {
+                console.error(err);
+                reject(err);
+              }
+              resolve(null);
+            },
+          );
+      });
+    };
+
+    //////////
+
+    const nc_isLinkExists = (airtableFieldId) => {
+      return !!ncLinkMappingTable.find(
+        (x) => x.aTbl.typeOptions.symmetricColumnId === airtableFieldId,
+      );
+    };
+
+    const nocoCreateProject = async (projName) => {
+      // create empty project (XC-DB)
+      logDetailed(`Create Project: ${projName}`);
+      const _perfStart = recordPerfStart();
+
+      ncCreatedProjectSchema = await this.projectsService.projectCreate({
+        project: { title: projName },
+        user: { id: syncDB.user.id },
+      });
+
+      recordPerfStats(_perfStart, 'project.create');
+    };
+
+    const nocoGetProject = async (projId) => {
+      // create empty project (XC-DB)
+      logDetailed(`Getting project meta: ${projId}`);
+      const _perfStart = recordPerfStart();
+      ncCreatedProjectSchema = await this.projectsService.getProjectWithInfo({
+        projectId: projId,
+      });
+      recordPerfStats(_perfStart, 'project.read');
+    };
+
+    const nocoConfigureGalleryView = async (sDB, aTblSchema) => {
+      if (!sDB.options.syncViews) return;
+      for (let idx = 0; idx < aTblSchema.length; idx++) {
+        const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
+        const galleryViews = aTblSchema[idx].views.filter(
+          (x) => x.type === 'gallery',
+        );
+
+        const configuredViews =
+          rtc.view.grid + rtc.view.gallery + rtc.view.form;
+        rtc.view.gallery += galleryViews.length;
+
+        for (let i = 0; i < galleryViews.length; i++) {
+          logDetailed(`   Axios fetch view-data`);
+
+          // create view
+          await getViewData(galleryViews[i].id);
+          const viewName = aTblSchema[idx].views.find(
+            (x) => x.id === galleryViews[i].id,
+          )?.name;
+
+          logBasic(
+            `:: [${configuredViews + i + 1}/${rtc.view.total}] Gallery : ${
+              aTblSchema[idx].name
+            } / ${viewName}`,
+          );
+
+          logDetailed(`NC API dbView.galleryCreate :: ${viewName}`);
+          const _perfStart = recordPerfStart();
+          await this.galleriesService.galleryViewCreate({
+            tableId: tblId,
+            gallery: {
+              title: viewName,
+            },
+          });
+          recordPerfStats(_perfStart, 'dbView.galleryCreate');
+
+          await updateNcTblSchemaById(tblId);
+        }
+      }
+    };
+
+    const nocoConfigureFormView = async (sDB, aTblSchema) => {
+      if (!sDB.options.syncViews) return;
+      for (let idx = 0; idx < aTblSchema.length; idx++) {
+        const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+        const formViews = aTblSchema[idx].views.filter(
+          (x) => x.type === 'form',
+        );
+
+        const configuredViews =
+          rtc.view.grid + rtc.view.gallery + rtc.view.form;
+        rtc.view.form += formViews.length;
+        for (let i = 0; i < formViews.length; i++) {
+          logDetailed(`   Axios fetch view-data`);
+
+          // create view
+          const vData = await getViewData(formViews[i].id);
+          const viewName = aTblSchema[idx].views.find(
+            (x) => x.id === formViews[i].id,
+          )?.name;
+
+          logBasic(
+            `:: [${configuredViews + i + 1}/${rtc.view.total}] Form : ${
+              aTblSchema[idx].name
+            } / ${viewName}`,
+          );
+
+          // everything is default
+          let refreshMode = 'NO_REFRESH';
+          let msg = 'Thank you for submitting the form!';
+          let desc = '';
+
+          // response will not include form object if everything is default
+          //
+          if (vData.metadata?.form) {
+            if (vData.metadata.form?.refreshAfterSubmit)
+              refreshMode = vData.metadata.form.refreshAfterSubmit;
+            if (vData.metadata.form?.afterSubmitMessage)
+              msg = vData.metadata.form.afterSubmitMessage;
+            if (vData.metadata.form?.description)
+              desc = vData.metadata.form.description;
+          }
+
+          const formData = {
+            title: viewName,
+            heading: viewName,
+            subheading: desc,
+            success_msg: msg,
+            submit_another_form: refreshMode.includes('REFRESH_BUTTON'),
+            show_blank_form: refreshMode.includes('AUTO_REFRESH'),
+          };
+
+          logDetailed(`NC API dbView.formCreate :: ${viewName}`);
+          const _perfStart = recordPerfStart();
+          // const f = await api.dbView.formCreate(tblId, formData);
+          const f = await this.formsService.formViewCreate({
+            tableId: tblId,
+            body: formData,
+          });
+          recordPerfStats(_perfStart, 'dbView.formCreate');
+
+          logDetailed(
+            `[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${
+              formViews.length
+            }] Create ${viewName}`,
+          );
+
+          await updateNcTblSchemaById(tblId);
+
+          logDetailed(`   Configure show/hide columns`);
+          await nc_configureFields(
+            f.id,
+            vData,
+            aTblSchema[idx].name,
+            viewName,
+            'form',
+          );
+        }
+      }
+    };
+
+    const nocoConfigureGridView = async (sDB, aTblSchema) => {
+      for (let idx = 0; idx < aTblSchema.length; idx++) {
+        const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+        const gridViews = aTblSchema[idx].views.filter(
+          (x) => x.type === 'grid',
+        );
+
+        let viewCnt = idx;
+        if (syncDB.options.syncViews)
+          viewCnt = rtc.view.grid + rtc.view.gallery + rtc.view.form;
+        rtc.view.grid += gridViews.length;
+
+        for (
+          let i = 0;
+          i < (sDB.options.syncViews ? gridViews.length : 1);
+          i++
+        ) {
+          logDetailed(`   Axios fetch view-data`);
+          // fetch viewData JSON
+          const vData = await getViewData(gridViews[i].id);
+
+          // retrieve view name & associated NC-ID
+          const viewName = aTblSchema[idx].views.find(
+            (x) => x.id === gridViews[i].id,
+          )?.name;
+          const _perfStart = recordPerfStart();
+          // const viewList: any = await api.dbView.list(tblId);
+          const viewList = { list: [] };
+          viewList['list'] = await this.viewsService.viewList({
+            tableId: tblId,
+            user: { roles: userRole },
+          });
+          recordPerfStats(_perfStart, 'dbView.list');
+
+          let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id;
+
+          logBasic(
+            `:: [${viewCnt + i + 1}/${rtc.view.total}] Grid : ${
+              aTblSchema[idx].name
+            } / ${viewName}`,
+          );
+
+          // create view (default already created)
+          if (i > 0) {
+            logDetailed(`NC API dbView.gridCreate :: ${viewName}`);
+            const _perfStart = recordPerfStart();
+            const viewCreated = await this.gridsService.gridViewCreate({
+              tableId: tblId,
+              grid: {
+                title: viewName,
+              },
+            });
+            recordPerfStats(_perfStart, 'dbView.gridCreate');
+
+            await updateNcTblSchemaById(tblId);
+            await sMap.addToMappingTbl(
+              gridViews[i].id,
+              viewCreated.id,
+              viewName,
+              tblId,
+            );
+            ncViewId = viewCreated.id;
+          }
+
+          logDetailed(`   Configure show/hide columns`);
+          await nc_configureFields(
+            ncViewId,
+            vData,
+            aTblSchema[idx].name,
+            viewName,
+            'grid',
+          );
+
+          // configure filters
+          if (vData?.filters) {
+            logDetailed(`   Configure filter set`);
+
+            // skip filters if nested
+            if (!vData.filters.filterSet.find((x) => x?.type === 'nested')) {
+              await nc_configureFilters(ncViewId, vData.filters);
+            }
+          }
+
+          // configure sort
+          if (vData?.lastSortsApplied?.sortSet.length) {
+            logDetailed(`   Configure sort set`);
+            await nc_configureSort(ncViewId, vData.lastSortsApplied);
+          }
+        }
+      }
+    };
+
+    const nocoAddUsers = async (aTblSchema) => {
+      const userRoles = {
+        owner: 'owner',
+        create: 'creator',
+        edit: 'editor',
+        comment: 'commenter',
+        read: 'viewer',
+        none: 'viewer',
+      };
+      const userList = aTblSchema.appBlanket.userInfoById;
+      const totalUsers = Object.keys(userList).length;
+      let cnt = 0;
+      const insertJobs: Promise<any>[] = [];
+
+      for (const [, value] of Object.entries(
+        userList as { [key: string]: any },
+      )) {
+        logDetailed(
+          `[${++cnt}/${totalUsers}] NC API auth.projectUserAdd :: ${
+            value.email
+          }`,
+        );
+        const _perfStart = recordPerfStart();
+        insertJobs.push(
+          this.projectUsersService
+            .userInvite({
+              projectId: ncCreatedProjectSchema.id,
+              projectUser: {
+                email: value.email,
+                roles: userRoles[value.permissionLevel],
+              },
+              req: { user: syncDB.user, clientIp: '' },
+            })
+            .catch((e) =>
+              e.response?.data?.msg
+                ? logBasic(`NOTICE: ${e.response.data.msg}`)
+                : console.log(e),
+            ),
+        );
+        recordPerfStats(_perfStart, 'auth.projectUserAdd');
+      }
+      await Promise.all(insertJobs);
+    };
+
+    const updateNcTblSchema = (tblSchema) => {
+      const tblId = tblSchema.id;
+
+      // replace entry from array if already exists
+      const idx = ncSchema.tables.findIndex((x) => x.id === tblId);
+      if (idx !== -1) ncSchema.tables.splice(idx, 1);
+      ncSchema.tables.push(tblSchema);
+
+      // overwrite object if it exists
+      ncSchema.tablesById[tblId] = tblSchema;
+    };
+
+    const updateNcTblSchemaById = async (tblId) => {
+      const _perfStart = recordPerfStart();
+      const ncTbl: any = await this.tablesService.getTableWithAccessibleViews({
+        tableId: tblId,
+        user: syncDB.user,
+      });
+      recordPerfStats(_perfStart, 'dbTable.read');
+
+      updateNcTblSchema(ncTbl);
+    };
+
+    ///////////////////////
+
+    // statistics
+    //
+    const migrationStats = [];
+
+    const generateMigrationStats = async (aTblSchema) => {
+      const migrationStatsObj = {
+        table_name: '',
+        aTbl: {
+          columns: 0,
+          links: 0,
+          lookup: 0,
+          rollup: 0,
+        },
+        nc: {
+          columns: 0,
+          links: 0,
+          lookup: 0,
+          rollup: 0,
+          invalidColumn: 0,
+        },
+      };
+      for (let idx = 0; idx < aTblSchema.length; idx++) {
+        migrationStatsObj.table_name = aTblSchema[idx].name;
+
+        const aTblLinkColumns = aTblSchema[idx].columns.filter(
+          (x) => x.type === 'foreignKey',
+        );
+        const aTblLookup = aTblSchema[idx].columns.filter(
+          (x) => x.type === 'lookup',
+        );
+        const aTblRollup = aTblSchema[idx].columns.filter(
+          (x) => x.type === 'rollup',
+        );
+
+        let invalidColumnId = 0;
+        for (let i = 0; i < aTblLookup.length; i++) {
+          if (
+            aTblLookup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length
+          ) {
+            invalidColumnId++;
+          }
+        }
+        for (let i = 0; i < aTblRollup.length; i++) {
+          if (
+            aTblRollup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length
+          ) {
+            invalidColumnId++;
+          }
+        }
+
+        migrationStatsObj.aTbl.columns = aTblSchema[idx].columns.length;
+        migrationStatsObj.aTbl.links = aTblLinkColumns.length;
+        migrationStatsObj.aTbl.lookup = aTblLookup.length;
+        migrationStatsObj.aTbl.rollup = aTblRollup.length;
+
+        const ncTbl = await nc_getTableSchema(aTblSchema[idx].name);
+        const linkColumn = ncTbl.columns.filter(
+          (x) => x.uidt === UITypes.LinkToAnotherRecord,
+        );
+        const lookup = ncTbl.columns.filter((x) => x.uidt === UITypes.Lookup);
+        const rollup = ncTbl.columns.filter((x) => x.uidt === UITypes.Rollup);
+
+        // all links hardwired as m2m. m2m generates additional tables per link
+        // hence link/2
+        migrationStatsObj.nc.columns =
+          ncTbl.columns.length - linkColumn.length / 2;
+        migrationStatsObj.nc.links = linkColumn.length / 2;
+        migrationStatsObj.nc.lookup = lookup.length;
+        migrationStatsObj.nc.rollup = rollup.length;
+        migrationStatsObj.nc.invalidColumn = invalidColumnId;
+
+        const temp = JSON.parse(JSON.stringify(migrationStatsObj));
+        migrationStats.push(temp);
+      }
+
+      const columnSum = migrationStats.reduce((accumulator, object) => {
+        return accumulator + object.nc.columns;
+      }, 0);
+      const linkSum = migrationStats.reduce((accumulator, object) => {
+        return accumulator + object.nc.links;
+      }, 0);
+      const lookupSum = migrationStats.reduce((accumulator, object) => {
+        return accumulator + object.nc.lookup;
+      }, 0);
+      const rollupSum = migrationStats.reduce((accumulator, object) => {
+        return accumulator + object.nc.rollup;
+      }, 0);
+
+      logBasic(`Quick Summary:`);
+      logBasic(`:: Total Tables:   ${aTblSchema.length}`);
+      logBasic(`:: Total Columns:  ${columnSum}`);
+      logBasic(`::   Links:        ${linkSum}`);
+      logBasic(`::   Lookup:       ${lookupSum}`);
+      logBasic(`::   Rollup:       ${rollupSum}`);
+      logBasic(`:: Total Filters:  ${rtc.filter}`);
+      logBasic(`:: Total Sort:     ${rtc.sort}`);
+      logBasic(`:: Total Views:    ${rtc.view.total}`);
+      logBasic(`::   Grid:         ${rtc.view.grid}`);
+      logBasic(`::   Gallery:      ${rtc.view.gallery}`);
+      logBasic(`::   Form:         ${rtc.view.form}`);
+      logBasic(`:: Total Records:  ${rtc.data.records}`);
+      logBasic(`:: Total Nested Links: ${rtc.data.nestedLinks}`);
+
+      const duration = Date.now() - start;
+      logBasic(`:: Migration time:      ${duration}`);
+      logBasic(`:: Axios fetch count:   ${rtc.fetchAt.count}`);
+      logBasic(`:: Axios fetch time:    ${rtc.fetchAt.time}`);
+
+      if (debugMode) {
+        await writeJsonFileAsync('stats.json', perfStats, { spaces: 2 });
+        const perflog = [];
+        for (let i = 0; i < perfStats.length; i++) {
+          perflog.push(`${perfStats[i].e}, ${perfStats[i].d}`);
+        }
+        await writeJsonFileAsync('stats.csv', perflog, { spaces: 2 });
+        await writeJsonFileAsync('skip.txt', rtc.migrationSkipLog.log, {
+          spaces: 2,
+        });
+      }
+
+      T.event({
+        event: 'a:airtable-import:success',
+        data: {
+          stats: {
+            migrationTime: duration,
+            totalTables: aTblSchema.length,
+            totalColumns: columnSum,
+            links: linkSum,
+            lookup: lookupSum,
+            rollup: rollupSum,
+            totalFilters: rtc.filter,
+            totalSort: rtc.sort,
+            view: {
+              total: rtc.view.total,
+              grid: rtc.view.grid,
+              gallery: rtc.view.gallery,
+              form: rtc.view.form,
+            },
+            axios: {
+              count: rtc.fetchAt.count,
+              time: rtc.fetchAt.time,
+            },
+            totalRecords: rtc.data.records,
+            nestedLinks: rtc.data.nestedLinks,
+          },
+        },
+      });
+    };
+
+    //////////////////////////////
+    // filters
+
+    const filterMap = {
+      '=': 'eq',
+      '!=': 'neq',
+      '<': 'lt',
+      '<=': 'lte',
+      '>': 'gt',
+      '>=': 'gte',
+      isEmpty: 'empty',
+      isNotEmpty: 'notempty',
+      contains: 'like',
+      doesNotContain: 'nlike',
+      isAnyOf: 'anyof',
+      isNoneOf: 'nanyof',
+      '|': 'anyof',
+      '&': 'allof',
+    };
+
+    const nc_configureFilters = async (viewId, f) => {
+      for (let i = 0; i < f.filterSet.length; i++) {
+        const filter = f.filterSet[i];
+        const colSchema = await nc_getColumnSchema(filter.columnId);
+
+        // column not available;
+        // one of not migrated column;
+        if (!colSchema) {
+          updateMigrationSkipLog(
+            await sMap.getNcNameFromAtId(viewId),
+            colSchema.title,
+            colSchema.uidt,
+            `filter config skipped; column not migrated`,
+          );
+          continue;
+        }
+        const columnId = colSchema.id;
+        const datatype = colSchema.uidt;
+        const ncFilters = [];
+
+        // console.log(filter)
+        if (datatype === UITypes.Date || datatype === UITypes.DateTime) {
+          // skip filters over data datatype
+          updateMigrationSkipLog(
+            await sMap.getNcNameFromAtId(viewId),
+            colSchema.title,
+            colSchema.uidt,
+            `filter config skipped; filter over date datatype not supported`,
+          );
+          continue;
+        }
+
+        // single-select & multi-select
+        else if (
+          datatype === UITypes.SingleSelect ||
+          datatype === UITypes.MultiSelect
+        ) {
+          if (filter.operator === 'doesNotContain') {
+            filter.operator = 'isNoneOf';
+          }
+          // if array, break it down to multiple filters
+          if (Array.isArray(filter.value)) {
+            const fx = {
+              fk_column_id: columnId,
+              logical_op: f.conjunction,
+              comparison_op: filterMap[filter.operator],
+              value: (
+                await Promise.all(
+                  filter.value.map(
+                    async (f) => await sMap.getNcNameFromAtId(f),
+                  ),
+                )
+              ).join(','),
+            };
+            ncFilters.push(fx);
+          }
+          // not array - add as is
+          else if (filter.value) {
+            const fx = {
+              fk_column_id: columnId,
+              logical_op: f.conjunction,
+              comparison_op: filterMap[filter.operator],
+              value: await sMap.getNcNameFromAtId(filter.value),
+            };
+            ncFilters.push(fx);
+          }
+        }
+
+        // other data types (number/ text/ long text/ ..)
+        else if (filter.value) {
+          const fx = {
+            fk_column_id: columnId,
+            logical_op: f.conjunction,
+            comparison_op: filterMap[filter.operator],
+            value: filter.value,
+          };
+          ncFilters.push(fx);
+        }
+
+        // insert filters
+        for (let i = 0; i < ncFilters.length; i++) {
+          const _perfStart = recordPerfStart();
+          await this.filtersService.filterCreate({
+            viewId: viewId,
+            filter: ncFilters[i],
+          });
+          recordPerfStats(_perfStart, 'dbTableFilter.create');
+
+          rtc.filter++;
+        }
+      }
+    };
+
+    const nc_configureSort = async (viewId, s) => {
+      for (let i = 0; i < s.sortSet.length; i++) {
+        const columnId = (await nc_getColumnSchema(s.sortSet[i].columnId))?.id;
+
+        if (columnId) {
+          const _perfStart = recordPerfStart();
+          await this.sortsService.sortCreate({
+            viewId: viewId,
+            sort: {
+              fk_column_id: columnId,
+              direction: s.sortSet[i].ascending ? 'asc' : 'desc',
+            },
+          });
+          recordPerfStats(_perfStart, 'dbTableSort.create');
+        }
+        rtc.sort++;
+      }
+    };
+
+    const nc_configureFields = async (
+      _viewId,
+      _c,
+      tblName,
+      viewName,
+      viewType?,
+    ) => {
+      // force hide PK column
+      const hiddenColumns = [ncSysFields.id, ncSysFields.hash];
+      const c = _c.columnOrder;
+
+      // column order corrections
+      // retrieve table schema
+      const ncTbl = await nc_getTableSchema(tblName);
+      // retrieve view ID
+      const viewId = ncTbl.views.find((x) => x.title === viewName).id;
+      let viewDetails;
+
+      const _perfStart = recordPerfStart();
+      if (viewType === 'form') {
+        viewDetails = (
+          await this.formsService.formViewGet({ formViewId: viewId })
+        ).columns;
+        recordPerfStats(_perfStart, 'dbView.formRead');
+      } else if (viewType === 'gallery') {
+        viewDetails = (
+          await this.galleriesService.galleryViewGet({
+            galleryViewId: viewId,
+          })
+        ).columns;
+        recordPerfStats(_perfStart, 'dbView.galleryRead');
+      } else {
+        viewDetails = await this.viewColumnsService.columnList({
+          viewId: viewId,
+        });
+        recordPerfStats(_perfStart, 'dbView.gridColumnsList');
+      }
+
+      // nc-specific columns; default hide.
+      for (let j = 0; j < hiddenColumns.length; j++) {
+        const ncColumnId = ncTbl.columns.find(
+          (x) => x.title === hiddenColumns[j],
+        ).id;
+        const ncViewColumnId = viewDetails.find(
+          (x) => x.fk_column_id === ncColumnId,
+        )?.id;
+        if (ncViewColumnId === undefined) continue;
+
+        // first two positions held by record id & record hash
+        const _perfStart = recordPerfStart();
+        await this.viewColumnsService.columnUpdate({
+          viewId: viewId,
+          columnId: ncViewColumnId,
+          column: {
+            show: false,
+            order: j + 1 + c.length,
+          },
+        });
+        recordPerfStats(_perfStart, 'dbViewColumn.update');
+      }
+
+      // rest of the columns from airtable- retain order & visibility property
+      for (let j = 0; j < c.length; j++) {
+        const ncColumnId = await sMap.getNcIdFromAtId(c[j].columnId);
+        const ncViewColumnId = await nc_getViewColumnId(
+          viewId,
+          viewType,
+          ncColumnId,
+        );
+        if (ncViewColumnId === undefined) continue;
+
+        // first two positions held by record id & record hash
+        const configData = { show: c[j].visibility, order: j + 1 };
+        if (viewType === 'form') {
+          if (_c?.metadata?.form?.fieldsByColumnId?.[c[j].columnId]) {
+            const x = _c.metadata.form.fieldsByColumnId[c[j].columnId];
+            const formData = { ...configData };
+            if (x?.title) formData[`label`] = x.title;
+            if (x?.required) formData[`required`] = x.required;
+            if (x?.description) formData[`description`] = x.description;
+            const _perfStart = recordPerfStart();
+            await this.formColumnsService.columnUpdate({
+              formViewColumnId: ncViewColumnId,
+              formViewColumn: formData,
+            });
+            recordPerfStats(_perfStart, 'dbView.formColumnUpdate');
+          }
+        }
+        const _perfStart = recordPerfStart();
+        await this.viewColumnsService.columnUpdate({
+          viewId: viewId,
+          columnId: ncViewColumnId,
+          column: configData,
+        });
+        recordPerfStats(_perfStart, 'dbViewColumn.update');
+      }
+    };
+
+    ///////////////////////////////////////////////////////////////////////////////
+    let recordCnt = 0;
+    try {
+      logBasic('SDK initialized');
+      logDetailed('Project initialization started');
+      // delete project if already exists
+      if (debugMode) await init(syncDB);
+
+      logDetailed('Project initialized');
+
+      logBasic('Retrieving Airtable schema');
+      // read schema file
+      const schema = await getAirtableSchema(syncDB);
+      const aTblSchema = schema.tableSchemas;
+      logDetailed('Project schema extraction completed');
+
+      if (!syncDB.projectId) {
+        if (!syncDB.projectName)
+          throw new Error('Project name or id not provided');
+        // create empty project
+        await nocoCreateProject(syncDB.projectName);
+        logDetailed('Project created');
+      } else {
+        await nocoGetProject(syncDB.projectId);
+        syncDB.projectName = ncCreatedProjectSchema?.title;
+        syncDB.baseId = syncDB.baseId || ncCreatedProjectSchema.bases[0].id;
+        logDetailed('Getting existing project meta');
+      }
+
+      logBasic('Importing Tables...');
+      // prepare table schema (base)
+      await nocoCreateBaseSchema(aTblSchema);
+      logDetailed('Table creation completed');
+
+      logDetailed('Configuring Links');
+      // add LTAR
+      await nocoCreateLinkToAnotherRecord(aTblSchema);
+      logDetailed('Migrating LTAR columns completed');
+
+      if (syncDB.options.syncLookup) {
+        logDetailed(`Configuring Lookup`);
+        // add look-ups
+        await nocoCreateLookups(aTblSchema);
+        logDetailed('Migrating Lookup columns completed');
+      }
+
+      if (syncDB.options.syncRollup) {
+        logDetailed('Configuring Rollup');
+        // add roll-ups
+        await nocoCreateRollup(aTblSchema);
+        logDetailed('Migrating Rollup columns completed');
+
+        if (syncDB.options.syncLookup) {
+          logDetailed('Migrating Lookup form Rollup columns');
+          // lookups for rollup
+          await nocoLookupForRollup();
+          logDetailed('Migrating Lookup form Rollup columns completed');
+        }
+      }
+      logDetailed('Configuring Display Value column');
+      // configure Display Value
+      await nocoSetPrimary(aTblSchema);
+      logDetailed('Configuring Display Value column completed');
+
+      logBasic('Configuring User(s)');
+      // add users
+      await nocoAddUsers(schema);
+      logDetailed('Adding users completed');
+
+      // hide-fields
+      // await nocoReconfigureFields(aTblSchema);
+
+      logBasic('Syncing views');
+      // configure views
+      await nocoConfigureGridView(syncDB, aTblSchema);
+      await nocoConfigureFormView(syncDB, aTblSchema);
+      await nocoConfigureGalleryView(syncDB, aTblSchema);
+      logDetailed('Syncing views completed');
+
+      if (syncDB.options.syncData) {
+        try {
+          // await nc_DumpTableSchema();
+          const _perfStart = recordPerfStart();
+          const ncTblList = { list: [] };
+          ncTblList['list'] = await this.tablesService.getAccessibleTables({
+            projectId: ncCreatedProjectSchema.id,
+            baseId: syncDB.baseId,
+            roles: userRole,
+          });
+          recordPerfStats(_perfStart, 'base.tableList');
+
+          logBasic('Reading Records...');
+
+          const recordsMap = {};
+
+          for (let i = 0; i < ncTblList.list.length; i++) {
+            // not a migrated table, skip
+            if (
+              undefined ===
+              aTblSchema.find((x) => x.name === ncTblList.list[i].title)
+            )
+              continue;
+
+            const _perfStart = recordPerfStart();
+            const ncTbl: any =
+              await this.tablesService.getTableWithAccessibleViews({
+                tableId: ncTblList.list[i].id,
+                user: syncDB.user,
+              });
+            recordPerfStats(_perfStart, 'dbTable.read');
+
+            recordCnt = 0;
+
+            recordsMap[ncTbl.id] = await importData({
+              projectName: syncDB.projectName,
+              table: ncTbl,
+              base,
+              logBasic,
+              nocoBaseDataProcessing_v2,
+              sDB: syncDB,
+              logDetailed,
+              services: {
+                tableService: this.tablesService,
+                bulkDataService: this.bulkDataAliasService,
+              },
+            });
+            rtc.data.records += await recordsMap[ncTbl.id].getCount();
+
+            logDetailed(`Data inserted from ${ncTbl.title}`);
+          }
+
+          logBasic('Configuring Record Links...');
+          for (let i = 0; i < ncTblList.list.length; i++) {
+            // not a migrated table, skip
+            if (
+              undefined ===
+              aTblSchema.find((x) => x.name === ncTblList.list[i].title)
+            )
+              continue;
+
+            // const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
+            const ncTbl: any =
+              await this.tablesService.getTableWithAccessibleViews({
+                tableId: ncTblList.list[i].id,
+                user: syncDB.user,
+              });
+
+            rtc.data.nestedLinks += await importLTARData({
+              table: ncTbl,
+              projectName: syncDB.projectName,
+              base,
+              fields: null, //Object.values(tblLinkGroup).flat(),
+              logBasic,
+              insertedAssocRef,
+              logDetailed,
+              records: recordsMap[ncTbl.id],
+              atNcAliasRef,
+              ncLinkMappingTable,
+              syncDB,
+              services: {
+                tableService: this.tablesService,
+                bulkDataService: this.bulkDataAliasService,
+              },
+            });
+          }
+        } catch (error) {
+          logDetailed(
+            `There was an error while migrating data! Please make sure your API key (${syncDB.apiKey}) is correct.`,
+          );
+          logDetailed(`Error: ${error}`);
+        }
+      }
+      if (generate_migrationStats) {
+        await generateMigrationStats(aTblSchema);
+      }
+    } catch (e) {
+      if (e.response?.data?.msg) {
+        T.event({
+          event: 'a:airtable-import:error',
+          data: { error: e.response.data.msg },
+        });
+        throw new Error(e.response.data.msg);
+      }
+      throw e;
+    }
+  }
+}
+
+export const getUniqueNameGenerator = (defaultName = 'name') => {
+  const namesRef = {};
+
+  return (initName: string = defaultName): string => {
+    let name = initName === '_' ? defaultName : initName;
+    let c = 0;
+    while (name in namesRef) {
+      name = `${initName}_${++c}`;
+    }
+    namesRef[name] = true;
+    return name;
+  };
+};
+
+export interface AirtableSyncConfig {
+  syncId: string;
+  baseURL: string;
+  authToken: string;
+  projectName?: string;
+  projectId?: string;
+  baseId?: string;
+  apiKey: string;
+  shareId: string;
+  user: UserType;
+  options: {
+    syncViews: boolean;
+    syncData: boolean;
+    syncRollup: boolean;
+    syncLookup: boolean;
+    syncFormula: boolean;
+    syncAttachment: boolean;
+  };
+}
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/helpers/EntityMap.ts b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/EntityMap.ts
new file mode 100644
index 0000000000..afbd1f0f53
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/EntityMap.ts
@@ -0,0 +1,222 @@
+import { Readable } from 'stream';
+import sqlite3 from 'sqlite3';
+
+class EntityMap {
+  initialized: boolean;
+  cols: string[];
+  db: any;
+
+  constructor(...args) {
+    this.initialized = false;
+    this.cols = args.map((arg) => processKey(arg));
+    this.db = new Promise((resolve, reject) => {
+      const db = new sqlite3.Database(':memory:');
+
+      const colStatement =
+        this.cols.length > 0
+          ? this.cols.join(' TEXT, ') + ' TEXT'
+          : 'mappingPlaceholder TEXT';
+      db.run(`CREATE TABLE mapping (${colStatement})`, (err) => {
+        if (err) {
+          console.log(err);
+          reject(err);
+        }
+        resolve(db);
+      });
+    });
+  }
+
+  async init() {
+    if (!this.initialized) {
+      this.db = await this.db;
+      this.initialized = true;
+    }
+  }
+
+  destroy() {
+    if (this.initialized && this.db) {
+      this.db.close();
+    }
+  }
+
+  async addRow(row) {
+    if (!this.initialized) {
+      throw 'Please initialize first!';
+    }
+
+    const cols = Object.keys(row).map((key) => processKey(key));
+    const colStatement = cols.map((key) => `'${key}'`).join(', ');
+    const questionMarks = cols.map(() => '?').join(', ');
+
+    const promises = [];
+
+    for (const col of cols.filter((col) => !this.cols.includes(col))) {
+      promises.push(
+        new Promise((resolve, reject) => {
+          this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => {
+            if (err) {
+              console.log(err);
+              reject(err);
+            }
+            this.cols.push(col);
+            resolve(true);
+          });
+        }),
+      );
+    }
+
+    await Promise.all(promises);
+
+    const values = Object.values(row).map((val) => {
+      if (typeof val === 'object') {
+        return `JSON::${JSON.stringify(val)}`;
+      }
+      return val;
+    });
+
+    return new Promise((resolve, reject) => {
+      this.db.run(
+        `INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`,
+        values,
+        (err) => {
+          if (err) {
+            console.log(err);
+            reject(err);
+          }
+          resolve(true);
+        },
+      );
+    });
+  }
+
+  getRow(col, val, res = []): Promise<Record<string, any>> {
+    if (!this.initialized) {
+      throw 'Please initialize first!';
+    }
+    return new Promise((resolve, reject) => {
+      col = processKey(col);
+      res = res.map((r) => processKey(r));
+      this.db.get(
+        `SELECT ${
+          res.length ? res.join(', ') : '*'
+        } FROM mapping WHERE ${col} = ?`,
+        [val],
+        (err, rs) => {
+          if (err) {
+            console.log(err);
+            reject(err);
+          }
+          if (rs) {
+            rs = processResponseRow(rs);
+          }
+          resolve(rs);
+        },
+      );
+    });
+  }
+
+  getCount(): Promise<number> {
+    if (!this.initialized) {
+      throw 'Please initialize first!';
+    }
+    return new Promise((resolve, reject) => {
+      this.db.get(`SELECT COUNT(*) as count FROM mapping`, (err, rs) => {
+        if (err) {
+          console.log(err);
+          reject(err);
+        }
+        resolve(rs.count);
+      });
+    });
+  }
+
+  getStream(res = []): DBStream {
+    if (!this.initialized) {
+      throw 'Please initialize first!';
+    }
+    res = res.map((r) => processKey(r));
+    return new DBStream(
+      this.db,
+      `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping`,
+    );
+  }
+
+  getLimit(limit, offset, res = []): Promise<Record<string, any>[]> {
+    if (!this.initialized) {
+      throw 'Please initialize first!';
+    }
+    return new Promise((resolve, reject) => {
+      res = res.map((r) => processKey(r));
+      this.db.all(
+        `SELECT ${
+          res.length ? res.join(', ') : '*'
+        } FROM mapping LIMIT ${limit} OFFSET ${offset}`,
+        (err, rs) => {
+          if (err) {
+            console.log(err);
+            reject(err);
+          }
+          for (let row of rs) {
+            row = processResponseRow(row);
+          }
+          resolve(rs);
+        },
+      );
+    });
+  }
+}
+
+class DBStream extends Readable {
+  db: any;
+  stmt: any;
+  sql: any;
+
+  constructor(db, sql) {
+    super({ objectMode: true });
+    this.db = db;
+    this.sql = sql;
+    this.stmt = this.db.prepare(this.sql);
+    this.on('end', () => this.stmt.finalize());
+  }
+
+  _read() {
+    const stream = this;
+    this.stmt.get(function (err, result) {
+      if (err) {
+        stream.emit('error', err);
+      } else {
+        if (result) {
+          result = processResponseRow(result);
+        }
+        stream.push(result || null);
+      }
+    });
+  }
+}
+
+function processResponseRow(res: any) {
+  for (const key of Object.keys(res)) {
+    if (res[key] && res[key].startsWith('JSON::')) {
+      try {
+        res[key] = JSON.parse(res[key].replace('JSON::', ''));
+      } catch (e) {
+        console.log(e);
+      }
+    }
+    if (revertKey(key) !== key) {
+      res[revertKey(key)] = res[key];
+      delete res[key];
+    }
+  }
+  return res;
+}
+
+function processKey(key) {
+  return key.replace(/'/g, "''").replace(/[A-Z]/g, (match) => `_${match}`);
+}
+
+function revertKey(key) {
+  return key.replace(/''/g, "'").replace(/_[A-Z]/g, (match) => match[1]);
+}
+
+export default EntityMap;
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/helpers/fetchAT.ts b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/fetchAT.ts
new file mode 100644
index 0000000000..1a9a8151bb
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/fetchAT.ts
@@ -0,0 +1,242 @@
+import axios from 'axios';
+
+const info: any = {
+  initialized: false,
+};
+
+async function initialize(shareId) {
+  info.cookie = '';
+  const url = `https://airtable.com/${shareId}`;
+
+  try {
+    const hreq = await axios
+      .get(url, {
+        headers: {
+          accept:
+            'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+          'accept-language': 'en-US,en;q=0.9',
+          'sec-ch-ua':
+            '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+          'sec-ch-ua-mobile': '?0',
+          'sec-ch-ua-platform': '"Linux"',
+          'sec-fetch-dest': 'document',
+          'sec-fetch-mode': 'navigate',
+          'sec-fetch-site': 'none',
+          'sec-fetch-user': '?1',
+          'upgrade-insecure-requests': '1',
+          'User-Agent':
+            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+        },
+        // @ts-ignore
+        referrerPolicy: 'strict-origin-when-cross-origin',
+        body: null,
+        method: 'GET',
+      })
+      .then((response) => {
+        for (const ck of response.headers['set-cookie']) {
+          info.cookie += ck.split(';')[0] + '; ';
+        }
+        return response.data;
+      })
+      .catch(() => {
+        throw {
+          message:
+            'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+        };
+      });
+
+    info.headers = JSON.parse(
+      hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
+    );
+    info.link = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim());
+    info.baseInfo = decodeURIComponent(info.link)
+      .match(/{(.*)}/g)[0]
+      .split('&')
+      .reduce((result, el) => {
+        try {
+          return Object.assign(
+            result,
+            JSON.parse(el.includes('=') ? el.split('=')[1] : el),
+          );
+        } catch (e) {
+          if (el.includes('=')) {
+            return Object.assign(result, {
+              [el.split('=')[0]]: el.split('=')[1],
+            });
+          }
+        }
+      }, {});
+    info.baseId = info.baseInfo.applicationId;
+    info.initialized = true;
+  } catch (e) {
+    console.log(e);
+    info.initialized = false;
+    if (e.message) {
+      throw e;
+    } else {
+      throw {
+        message:
+          'Error processing Shared Base :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+      };
+    }
+  }
+}
+
+async function read() {
+  if (info.initialized) {
+    const resreq = await axios('https://airtable.com' + info.link, {
+      headers: {
+        accept: '*/*',
+        'accept-language': 'en-US,en;q=0.9',
+        'sec-ch-ua':
+          '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+        'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua-platform': '"Linux"',
+        'sec-fetch-dest': 'empty',
+        'sec-fetch-mode': 'cors',
+        'sec-fetch-site': 'same-origin',
+        'User-Agent':
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+        'x-time-zone': 'Europe/Berlin',
+        cookie: info.cookie,
+        ...info.headers,
+      },
+      // @ts-ignore
+      referrerPolicy: 'no-referrer',
+      body: null,
+      method: 'GET',
+    })
+      .then((response) => {
+        return response.data;
+      })
+      .catch(() => {
+        throw {
+          message:
+            'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+        };
+      });
+
+    return {
+      schema: resreq.data,
+      baseId: info.baseId,
+      baseInfo: info.baseInfo,
+    };
+  } else {
+    throw {
+      message: 'Error Initializing :: please try again !!',
+    };
+  }
+}
+
+async function readView(viewId) {
+  if (info.initialized) {
+    const resreq = await axios(
+      `https://airtable.com/v0.3/view/${viewId}/readData?` +
+        `stringifiedObjectParams=${encodeURIComponent('{}')}&requestId=${
+          info.baseInfo.requestId
+        }&accessPolicy=${encodeURIComponent(
+          JSON.stringify({
+            allowedActions: info.baseInfo.allowedActions,
+            shareId: info.baseInfo.shareId,
+            applicationId: info.baseInfo.applicationId,
+            generationNumber: info.baseInfo.generationNumber,
+            expires: info.baseInfo.expires,
+            signature: info.baseInfo.signature,
+          }),
+        )}`,
+      {
+        headers: {
+          accept: '*/*',
+          'accept-language': 'en-US,en;q=0.9',
+          'sec-ch-ua':
+            '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+          'sec-ch-ua-mobile': '?0',
+          'sec-ch-ua-platform': '"Linux"',
+          'sec-fetch-dest': 'empty',
+          'sec-fetch-mode': 'cors',
+          'sec-fetch-site': 'same-origin',
+          'User-Agent':
+            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+          'x-time-zone': 'Europe/Berlin',
+          cookie: info.cookie,
+          ...info.headers,
+        },
+        // @ts-ignore
+        referrerPolicy: 'no-referrer',
+        body: null,
+        method: 'GET',
+      },
+    )
+      .then((response) => {
+        return response.data;
+      })
+      .catch(() => {
+        throw {
+          message:
+            'Error Reading View :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+        };
+      });
+    return { view: resreq.data };
+  } else {
+    throw {
+      message: 'Error Initializing :: please try again !!',
+    };
+  }
+}
+
+async function readTemplate(templateId) {
+  if (!info.initialized) {
+    await initialize('shrO8aYf3ybwSdDKn');
+  }
+  const resreq = await axios(
+    `https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
+    {
+      headers: {
+        accept: '*/*',
+        'accept-language': 'en-US,en;q=0.9',
+        'sec-ch-ua':
+          '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+        'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua-platform': '"Linux"',
+        'sec-fetch-dest': 'empty',
+        'sec-fetch-mode': 'cors',
+        'sec-fetch-site': 'same-origin',
+        'User-Agent':
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+        'x-time-zone': 'Europe/Berlin',
+        cookie: info.cookie,
+        ...info.headers,
+      },
+      // @ts-ignore
+      referrer: 'https://www.airtable.com/',
+      referrerPolicy: 'same-origin',
+      body: null,
+      method: 'GET',
+      mode: 'cors',
+      credentials: 'include',
+    },
+  )
+    .then((response) => {
+      return response.data;
+    })
+    .catch(() => {
+      throw {
+        message:
+          'Error Fetching :: Ensure www.airtable.com/templates/featured/<TemplateID> is accessible.',
+      };
+    });
+  return { template: resreq };
+}
+
+function unicodeToChar(text) {
+  return text.replace(/\\u[\dA-F]{4}/gi, function (match) {
+    return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16));
+  });
+}
+
+export default {
+  initialize,
+  read,
+  readView,
+  readTemplate,
+};
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/helpers/readAndProcessData.ts b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/readAndProcessData.ts
new file mode 100644
index 0000000000..7de16fad7e
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/readAndProcessData.ts
@@ -0,0 +1,362 @@
+/* eslint-disable no-async-promise-executor */
+import { RelationTypes, UITypes } from 'nocodb-sdk';
+import EntityMap from './EntityMap';
+import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
+import type { TablesService } from '../../../../services/tables.service';
+// @ts-ignore
+import type { AirtableBase } from 'airtable/lib/airtable_base';
+import type { TableType } from 'nocodb-sdk';
+
+const BULK_DATA_BATCH_SIZE = 500;
+const ASSOC_BULK_DATA_BATCH_SIZE = 1000;
+const BULK_PARALLEL_PROCESS = 5;
+
+interface AirtableImportContext {
+  bulkDataService: BulkDataAliasService;
+  tableService: TablesService;
+}
+
+async function readAllData({
+  table,
+  fields,
+  base,
+  logBasic = (_str) => {},
+  services,
+}: {
+  table: { title?: string };
+  fields?;
+  base: AirtableBase;
+  logBasic?: (string) => void;
+  logDetailed?: (string) => void;
+  services: AirtableImportContext;
+}): Promise<EntityMap> {
+  return new Promise((resolve, reject) => {
+    let data = null;
+
+    const selectParams: any = {
+      pageSize: 100,
+    };
+
+    if (fields) selectParams.fields = fields;
+
+    base(table.title)
+      .select(selectParams)
+      .eachPage(
+        async function page(records, fetchNextPage) {
+          if (!data) {
+            data = new EntityMap();
+            await data.init();
+          }
+
+          for await (const record of records) {
+            await data.addRow({ id: record.id, ...record.fields });
+          }
+
+          const tmpLength = await data.getCount();
+
+          logBasic(
+            `:: Reading '${table.title}' data :: ${Math.max(
+              1,
+              tmpLength - records.length,
+            )} - ${tmpLength}`,
+          );
+
+          // To fetch the next page of records, call `fetchNextPage`.
+          // If there are more records, `page` will get called again.
+          // If there are no more records, `done` will get called.
+          fetchNextPage();
+        },
+        async function done(err) {
+          if (err) {
+            console.error(err);
+            return reject(err);
+          }
+          resolve(data);
+        },
+      );
+  });
+}
+
+export async function importData({
+  projectName,
+  table,
+  base,
+  nocoBaseDataProcessing_v2,
+  sDB,
+  logDetailed = (_str) => {},
+  logBasic = (_str) => {},
+  services,
+}: {
+  projectName: string;
+  table: { title?: string; id?: string };
+  fields?;
+  base: AirtableBase;
+  logBasic: (string) => void;
+  logDetailed: (string) => void;
+  nocoBaseDataProcessing_v2;
+  sDB;
+  services: AirtableImportContext;
+}): Promise<EntityMap> {
+  try {
+    // @ts-ignore
+    const records = await readAllData({
+      table,
+      base,
+      logDetailed,
+      logBasic,
+    });
+
+    await new Promise(async (resolve) => {
+      const readable = records.getStream();
+      const allRecordsCount = await records.getCount();
+      const promises = [];
+      let tempData = [];
+      let importedCount = 0;
+      let activeProcess = 0;
+      readable.on('data', async (record) => {
+        promises.push(
+          new Promise(async (resolve) => {
+            activeProcess++;
+            if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause();
+            const { id: rid, ...fields } = record;
+            const r = await nocoBaseDataProcessing_v2(sDB, table, {
+              id: rid,
+              fields,
+            });
+            tempData.push(r);
+
+            if (tempData.length >= BULK_DATA_BATCH_SIZE) {
+              let insertArray = tempData.splice(0, tempData.length);
+
+              await services.bulkDataService.bulkDataInsert({
+                projectName,
+                tableName: table.title,
+                body: insertArray,
+                cookie: {},
+              });
+
+              logBasic(
+                `:: Importing '${
+                  table.title
+                }' data :: ${importedCount} - ${Math.min(
+                  importedCount + BULK_DATA_BATCH_SIZE,
+                  allRecordsCount,
+                )}`,
+              );
+              importedCount += insertArray.length;
+              insertArray = [];
+            }
+            activeProcess--;
+            if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume();
+            resolve(true);
+          }),
+        );
+      });
+      readable.on('end', async () => {
+        await Promise.all(promises);
+        if (tempData.length > 0) {
+          await services.bulkDataService.bulkDataInsert({
+            projectName,
+            tableName: table.title,
+            body: tempData,
+            cookie: {},
+          });
+
+          logBasic(
+            `:: Importing '${
+              table.title
+            }' data :: ${importedCount} - ${Math.min(
+              importedCount + BULK_DATA_BATCH_SIZE,
+              allRecordsCount,
+            )}`,
+          );
+          importedCount += tempData.length;
+          tempData = [];
+        }
+        resolve(true);
+      });
+    });
+
+    return records;
+  } catch (e) {
+    console.log(e);
+    return null;
+  }
+}
+
+export async function importLTARData({
+  table,
+  fields,
+  base,
+  projectName,
+  insertedAssocRef = {},
+  logDetailed = (_str) => {},
+  logBasic = (_str) => {},
+  records,
+  atNcAliasRef,
+  ncLinkMappingTable,
+  syncDB,
+  services,
+}: {
+  projectName: string;
+  table: { title?: string; id?: string };
+  fields;
+  base: AirtableBase;
+  logDetailed: (string) => void;
+  logBasic: (string) => void;
+  insertedAssocRef: { [assocTableId: string]: boolean };
+  records?: EntityMap;
+  atNcAliasRef: {
+    [ncTableId: string]: {
+      [ncTitle: string]: string;
+    };
+  };
+  ncLinkMappingTable: Record<string, Record<string, any>>[];
+  syncDB;
+  services: AirtableImportContext;
+}) {
+  const assocTableMetas: Array<{
+    modelMeta: { id?: string; title?: string };
+    colMeta: { title?: string };
+    curCol: { title?: string };
+    refCol: { title?: string };
+  }> = [];
+  const allData =
+    records ||
+    (await readAllData({
+      table,
+      fields,
+      base,
+      logDetailed,
+      logBasic,
+      services,
+    }));
+
+  const modelMeta: any =
+    await services.tableService.getTableWithAccessibleViews({
+      tableId: table.id,
+      user: syncDB.user,
+    });
+
+  for (const colMeta of modelMeta.columns) {
+    // skip columns which are not LTAR and Many to many
+    if (
+      colMeta.uidt !== UITypes.LinkToAnotherRecord ||
+      colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY
+    ) {
+      continue;
+    }
+
+    // skip if already inserted
+    if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue;
+
+    // self links: skip if the column under consideration is the add-on column NocoDB creates
+    if (ncLinkMappingTable.every((a) => a.nc.title !== colMeta.title)) continue;
+
+    // mark as inserted
+    insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true;
+
+    const assocModelMeta: TableType =
+      (await services.tableService.getTableWithAccessibleViews({
+        tableId: colMeta.colOptions.fk_mm_model_id,
+        user: syncDB.user,
+      })) as any;
+
+    // extract associative table and columns meta
+    assocTableMetas.push({
+      modelMeta: assocModelMeta,
+      colMeta,
+      curCol: assocModelMeta.columns.find(
+        (c) => c.id === colMeta.colOptions.fk_mm_child_column_id,
+      ),
+      refCol: assocModelMeta.columns.find(
+        (c) => c.id === colMeta.colOptions.fk_mm_parent_column_id,
+      ),
+    });
+  }
+
+  let nestedLinkCnt = 0;
+  // Iterate over all related M2M associative  table
+  for await (const assocMeta of assocTableMetas) {
+    let assocTableData = [];
+    let importedCount = 0;
+
+    //  extract insert data from records
+    await new Promise((resolve) => {
+      const promises = [];
+      const readable = allData.getStream();
+      let activeProcess = 0;
+      readable.on('data', async (record) => {
+        promises.push(
+          new Promise(async (resolve) => {
+            activeProcess++;
+            if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause();
+            const { id: _atId, ...rec } = record;
+
+            // todo: use actual alias instead of sanitized
+            assocTableData.push(
+              ...(
+                rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || []
+              ).map((id) => ({
+                [assocMeta.curCol.title]: record.id,
+                [assocMeta.refCol.title]: id,
+              })),
+            );
+
+            if (assocTableData.length >= ASSOC_BULK_DATA_BATCH_SIZE) {
+              let insertArray = assocTableData.splice(0, assocTableData.length);
+              logBasic(
+                `:: Importing '${
+                  table.title
+                }' LTAR data :: ${importedCount} - ${Math.min(
+                  importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
+                  insertArray.length,
+                )}`,
+              );
+
+              await services.bulkDataService.bulkDataInsert({
+                projectName,
+                tableName: assocMeta.modelMeta.title,
+                body: insertArray,
+                cookie: {},
+              });
+
+              importedCount += insertArray.length;
+              insertArray = [];
+            }
+            activeProcess--;
+            if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume();
+            resolve(true);
+          }),
+        );
+      });
+      readable.on('end', async () => {
+        await Promise.all(promises);
+        if (assocTableData.length >= 0) {
+          logBasic(
+            `:: Importing '${
+              table.title
+            }' LTAR data :: ${importedCount} - ${Math.min(
+              importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
+              assocTableData.length,
+            )}`,
+          );
+
+          await services.bulkDataService.bulkDataInsert({
+            projectName,
+            tableName: assocMeta.modelMeta.title,
+            body: assocTableData,
+            cookie: {},
+          });
+
+          importedCount += assocTableData.length;
+          assocTableData = [];
+        }
+        resolve(true);
+      });
+    });
+
+    nestedLinkCnt += importedCount;
+  }
+  return nestedLinkCnt;
+}
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/helpers/syncMap.ts b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/syncMap.ts
new file mode 100644
index 0000000000..79ea61511c
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/syncMap.ts
@@ -0,0 +1,31 @@
+export const mapTbl = {};
+
+// static mapping records between aTblId && ncId
+export const addToMappingTbl = function addToMappingTbl(
+  aTblId,
+  ncId,
+  ncName,
+  parent?,
+) {
+  mapTbl[aTblId] = {
+    ncId: ncId,
+    ncParent: parent,
+    // name added to assist in quick debug
+    ncName: ncName,
+  };
+};
+
+// get NcID from airtable ID
+export const getNcIdFromAtId = function getNcIdFromAtId(aId) {
+  return mapTbl[aId]?.ncId;
+};
+
+// get nc Parent from airtable ID
+export const getNcParentFromAtId = function getNcParentFromAtId(aId) {
+  return mapTbl[aId]?.ncParent;
+};
+
+// get nc-title from airtable ID
+export const getNcNameFromAtId = function getNcNameFromAtId(aId) {
+  return mapTbl[aId]?.ncName;
+};
diff --git a/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts b/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts
index 3ac9ced9ff..bd3c9ee4c5 100644
--- a/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts
+++ b/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts
@@ -3,6 +3,7 @@ import PQueue from 'p-queue';
 import Emittery from 'emittery';
 import { DuplicateProcessor } from './export-import/duplicate.processor';
 import { JobsEventService } from './jobs-event.service';
+import { AtImportProcessor } from './at-import/at-import.processor';
 
 interface Job {
   id: string;
@@ -22,6 +23,7 @@ export class QueueService {
   constructor(
     private readonly jobsEventService: JobsEventService,
     private readonly duplicateProcessor: DuplicateProcessor,
+    private readonly atImportProcessor: AtImportProcessor,
   ) {
     this.emitter.on('active', (data: any) => {
       const job = this.queueMemory.find(
@@ -56,6 +58,10 @@ export class QueueService {
       this: this.duplicateProcessor,
       fn: this.duplicateProcessor.duplicateBase,
     },
+    'at-import': {
+      this: this.atImportProcessor,
+      fn: this.atImportProcessor.job,
+    },
   };
 
   async jobWrapper(job: Job) {
diff --git a/packages/nocodb-nest/src/modules/jobs/jobs-event.service.ts b/packages/nocodb-nest/src/modules/jobs/jobs-event.service.ts
index dfc400e4e7..106fc1547f 100644
--- a/packages/nocodb-nest/src/modules/jobs/jobs-event.service.ts
+++ b/packages/nocodb-nest/src/modules/jobs/jobs-event.service.ts
@@ -38,6 +38,7 @@ export class JobsEventService {
       name: job.name,
       id: job.id.toString(),
       status: 'failed',
+      error: error?.message,
     });
   }
 
diff --git a/packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts b/packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts
index f9b521afa1..c7c3420777 100644
--- a/packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts
+++ b/packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts
@@ -40,10 +40,16 @@ export class JobsGateway implements OnModuleInit {
 
   @SubscribeMessage('subscribe')
   async subscribe(
-    @MessageBody() data: { name: string; id: string } | { id: string },
+    @MessageBody()
+    body: { _id: number; data: { id: string; name: string } | any },
     @ConnectedSocket() client: Socket,
   ): Promise<void> {
-    if ('name' in data) {
+    const { _id, data } = body;
+    if (
+      Object.keys(data).every((k) => ['name', 'id'].includes(k)) &&
+      data?.name &&
+      data?.id
+    ) {
       const rooms = (await this.jobsService.jobList(data.name)).map(
         (j) => `${j.name}-${j.id}`,
       );
@@ -51,17 +57,17 @@ export class JobsGateway implements OnModuleInit {
       if (room) {
         client.join(`${data.name}-${data.id}`);
         client.emit('subscribed', {
-          subbed: data.id,
+          _id,
           name: data.name,
           id: data.id,
         });
       }
     } else {
-      const job = await this.jobsService.getJobWithData({ id: data.id });
+      const job = await this.jobsService.getJobWithData(data);
       if (job) {
         client.join(`${job.name}-${job.id}`);
         client.emit('subscribed', {
-          subbed: data.id,
+          _id,
           name: job.name,
           id: job.id,
         });
@@ -71,10 +77,12 @@ export class JobsGateway implements OnModuleInit {
 
   @SubscribeMessage('status')
   async status(
-    @MessageBody() data: { name: string; id: string },
+    @MessageBody() body: { _id: number; data: { id: string; name: string } },
     @ConnectedSocket() client: Socket,
   ): Promise<void> {
+    const { _id, data } = body;
     client.emit('status', {
+      _id,
       id: data.id,
       name: data.name,
       status: await this.jobsService.jobStatus(data.id),
@@ -93,11 +101,13 @@ export class JobsGateway implements OnModuleInit {
       | 'failed'
       | 'paused'
       | 'refresh';
+    error?: any;
   }): Promise<void> {
     this.server.to(`${data.name}-${data.id}`).emit('status', {
       id: data.id,
       name: data.name,
       status: data.status,
+      error: data.error,
     });
   }
 
diff --git a/packages/nocodb-nest/src/modules/jobs/jobs.module.ts b/packages/nocodb-nest/src/modules/jobs/jobs.module.ts
index 33c8b81471..13de39a065 100644
--- a/packages/nocodb-nest/src/modules/jobs/jobs.module.ts
+++ b/packages/nocodb-nest/src/modules/jobs/jobs.module.ts
@@ -11,6 +11,8 @@ import { DuplicateProcessor } from './export-import/duplicate.processor';
 import { JobsGateway } from './jobs.gateway';
 import { QueueService } from './fallback-queue.service';
 import { JobsEventService } from './jobs-event.service';
+import { AtImportController } from './at-import/at-import.controller';
+import { AtImportProcessor } from './at-import/at-import.processor';
 
 @Module({
   imports: [
@@ -21,7 +23,7 @@ import { JobsEventService } from './jobs-event.service';
       name: 'jobs',
     }),
   ],
-  controllers: [DuplicateController],
+  controllers: [DuplicateController, AtImportController],
   providers: [
     QueueService,
     JobsGateway,
@@ -30,6 +32,7 @@ import { JobsEventService } from './jobs-event.service';
     DuplicateProcessor,
     ExportService,
     ImportService,
+    AtImportProcessor,
   ],
 })
 export class JobsModule {}
diff --git a/packages/nocodb-nest/src/modules/jobs/jobs.service.ts b/packages/nocodb-nest/src/modules/jobs/jobs.service.ts
index 3271f631bb..dc0abed3c2 100644
--- a/packages/nocodb-nest/src/modules/jobs/jobs.service.ts
+++ b/packages/nocodb-nest/src/modules/jobs/jobs.service.ts
@@ -21,17 +21,17 @@ export class JobsService {
 
   async jobList(jobType: string) {
     return (
-      await this.activeQueue.getJobs(['active', 'waiting', 'delayed'])
+      await this.activeQueue.getJobs(['active', 'waiting', 'delayed', 'paused'])
     ).filter((j) => j.name === jobType);
   }
 
   async getJobWithData(data: any) {
     const jobs = await this.activeQueue.getJobs([
-      'completed',
+      // 'completed',
       'waiting',
       'active',
       'delayed',
-      'failed',
+      // 'failed',
       'paused',
     ]);
 
diff --git a/packages/nocodb-nest/src/modules/metas/metas.module.ts b/packages/nocodb-nest/src/modules/metas/metas.module.ts
index 1753b278e3..75f96f75cc 100644
--- a/packages/nocodb-nest/src/modules/metas/metas.module.ts
+++ b/packages/nocodb-nest/src/modules/metas/metas.module.ts
@@ -16,7 +16,6 @@ import { GalleriesController } from '../../controllers/galleries.controller';
 import { GridColumnsController } from '../../controllers/grid-columns.controller';
 import { GridsController } from '../../controllers/grids.controller';
 import { HooksController } from '../../controllers/hooks.controller';
-import { ImportController } from '../../controllers/imports/import.controller';
 import { KanbansController } from '../../controllers/kanbans.controller';
 import { MapsController } from '../../controllers/maps.controller';
 import { MetaDiffsController } from '../../controllers/meta-diffs.controller';
@@ -98,7 +97,6 @@ import { DatasModule } from '../datas/datas.module';
     GridColumnsController,
     GridsController,
     HooksController,
-    ImportController,
     KanbansController,
     MapsController,
     MetaDiffsController,
@@ -170,6 +168,8 @@ import { DatasModule } from '../datas/datas.module';
     GalleriesService,
     KanbansService,
     ProjectsService,
+    AttachmentsService,
+    ProjectUsersService,
   ],
 })
 export class MetasModule {}
diff --git a/packages/nocodb-nest/src/services/socket.gateway.ts b/packages/nocodb-nest/src/services/socket.gateway.ts
index c621847de6..192c24caaa 100644
--- a/packages/nocodb-nest/src/services/socket.gateway.ts
+++ b/packages/nocodb-nest/src/services/socket.gateway.ts
@@ -25,7 +25,6 @@ function getHash(str) {
 export class SocketGateway implements OnModuleInit {
   // private server: HttpServer;
   private clients: { [id: string]: Socket } = {};
-  private _jobs: { [id: string]: { last_message: any } } = {};
 
   constructor(
     private jwtStrategy: JwtStrategy,
@@ -59,21 +58,10 @@ export class SocketGateway implements OnModuleInit {
         socket.on('event', (args) => {
           T.event({ ...args, id });
         });
-        socket.on('subscribe', (room) => {
-          if (room in this.jobs) {
-            socket.join(room);
-            socket.emit('job');
-            socket.emit('progress', this.jobs[room].last_message);
-          }
-        });
       });
   }
 
   public get io() {
     return this.server;
   }
-
-  public get jobs() {
-    return this._jobs;
-  }
 }