feat: plugi, caches apis

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb-nest/src/app.module.ts

@@ -41,9 +41,14 @@ import { ApiDocsModule } from './modules/api-docs/api-docs.module';
 import { PublicMetasModule } from './modules/public-metas/public-metas.module';
 import { PublicDatasModule } from './modules/public-datas/public-datas.module';
 import { PublicDatasExportModule } from './modules/public-datas-export/public-datas-export.module';
+import { SyncModule } from './modules/sync/sync.module';
+import { ImportModule } from './modules/import/import.module';
+import { CachesModule } from './modules/caches/caches.module';
+import { TestModule } from './modules/test/test.module';
+import { PluginsModule } from './modules/plugins/plugins.module';
 
 @Module({
-  imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule, AuditsModule, DatasModule, DataAliasModule, ApiDocsModule, PublicMetasModule, PublicDatasModule, PublicDatasExportModule],
+  imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule, AuditsModule, DatasModule, DataAliasModule, ApiDocsModule, PublicMetasModule, PublicDatasModule, PublicDatasExportModule, SyncModule, ImportModule, CachesModule, TestModule, PluginsModule],
   controllers: [],
   providers: [Connection, MetaService, JwtStrategy, ExtractProjectIdMiddleware],
   exports: [Connection, MetaService],

packages/nocodb-nest/src/modules/caches/caches.controller.spec.ts

@@ -0,0 +1,20 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { CachesController } from './caches.controller';
+import { CachesService } from './caches.service';
+
+describe('CachesController', () => {
+  let controller: CachesController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [CachesController],
+      providers: [CachesService],
+    }).compile();
+
+    controller = module.get<CachesController>(CachesController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/caches/caches.controller.ts

@@ -0,0 +1,7 @@
+import { Controller } from '@nestjs/common';
+import { CachesService } from './caches.service';
+
+@Controller('caches')
+export class CachesController {
+  constructor(private readonly cachesService: CachesService) {}
+}

packages/nocodb-nest/src/modules/caches/caches.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { CachesService } from './caches.service';
+import { CachesController } from './caches.controller';
+
+@Module({
+  controllers: [CachesController],
+  providers: [CachesService]
+})
+export class CachesModule {}

packages/nocodb-nest/src/modules/caches/caches.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { CachesService } from './caches.service';
+
+describe('CachesService', () => {
+  let service: CachesService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [CachesService],
+    }).compile();
+
+    service = module.get<CachesService>(CachesService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/caches/caches.service.ts

@@ -0,0 +1,4 @@
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class CachesService {}

packages/nocodb-nest/src/modules/import/import.controller.spec.ts

@@ -0,0 +1,20 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { ImportController } from './import.controller';
+import { ImportService } from './import.service';
+
+describe('ImportController', () => {
+  let controller: ImportController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [ImportController],
+      providers: [ImportService],
+    }).compile();
+
+    controller = module.get<ImportController>(ImportController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/import/import.controller.ts

@@ -0,0 +1,142 @@
+import { Controller } from '@nestjs/common';
+import { ImportService } from './import.service';
+
+const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB';
+const AIRTABLE_PROGRESS_JOB = 'AIRTABLE_PROGRESS_JOB';
+
+enum SyncStatus {
+  PROGRESS = 'PROGRESS',
+  COMPLETED = 'COMPLETED',
+  FAILED = 'FAILED',
+}
+
+@Controller('import')
+export class ImportController {
+  constructor(private readonly importService: ImportService) {}
+
+
+
+
+ default (
+  router: Router,
+  sv: Server,
+  jobs: { [id: string]: { last_message: any } }
+) => {
+  // add importer job handler and progress notification job handler
+  NocoJobs.jobsMgr.addJobWorker(
+    AIRTABLE_IMPORT_JOB,
+    syncService.airtableImportJob
+  );
+  NocoJobs.jobsMgr.addJobWorker(
+    AIRTABLE_PROGRESS_JOB,
+    ({ payload, progress }) => {
+      sv.to(payload?.id).emit('progress', {
+        msg: progress?.msg,
+        level: progress?.level,
+        status: progress?.status,
+      });
+
+      if (payload?.id in jobs) {
+        jobs[payload?.id].last_message = {
+          msg: progress?.msg,
+          level: progress?.level,
+          status: progress?.status,
+        };
+      }
+    }
+  );
+
+  NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => {
+    NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, {
+      payload,
+      progress: {
+        msg: progress?.msg,
+        level: progress?.level,
+        status: progress?.status,
+      },
+    });
+  });
+  NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, (payload) => {
+    NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, {
+      payload,
+      progress: {
+        msg: 'Complete!',
+        status: SyncStatus.COMPLETED,
+      },
+    });
+    delete jobs[payload?.id];
+  });
+  NocoJobs.jobsMgr.addFailureCbk(AIRTABLE_IMPORT_JOB, (payload, error: any) => {
+    NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, {
+      payload,
+      progress: {
+        msg: error?.message || 'Failed due to some internal error',
+        status: SyncStatus.FAILED,
+      },
+    });
+    delete jobs[payload?.id];
+  });
+
+  router.post(
+    '/api/v1/db/meta/import/airtable',
+    catchError((req, res) => {
+      NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, {
+        id: req.query.id,
+        ...req.body,
+      });
+      res.json({});
+    })
+  );
+  router.post(
+    '/api/v1/db/meta/syncs/:syncId/trigger',
+    catchError(async (req: Request, res) => {
+      if (req.params.syncId in jobs) {
+        NcError.badRequest('Sync already in progress');
+      }
+
+      const syncSource = await SyncSource.get(req.params.syncId);
+
+      const user = await syncSource.getUser();
+      const token = userService.genJwt(user, Noco.getConfig());
+
+      // Treat default baseUrl as siteUrl from req object
+      let baseURL = (req as any).ncSiteUrl;
+
+      // if environment value avail use it
+      // or if it's docker construct using `PORT`
+      if (process.env.NC_DOCKER) {
+        baseURL = `http://localhost:${process.env.PORT || 8080}`;
+      }
+
+      setTimeout(() => {
+        NocoJobs.jobsMgr.add<AirtableSyncConfig>(AIRTABLE_IMPORT_JOB, {
+          id: req.params.syncId,
+          ...(syncSource?.details || {}),
+          projectId: syncSource.project_id,
+          baseId: syncSource.base_id,
+          authToken: token,
+          baseURL,
+          user: user,
+        });
+      }, 1000);
+
+      jobs[req.params.syncId] = {
+        last_message: {
+          msg: 'Sync started',
+        },
+      };
+      res.json({});
+    })
+  );
+  router.post(
+    '/api/v1/db/meta/syncs/:syncId/abort',
+    catchError(async (req: Request, res) => {
+      if (req.params.syncId in jobs) {
+        delete jobs[req.params.syncId];
+      }
+      res.json({});
+    })
+  );
+};
+
+}

packages/nocodb-nest/src/modules/import/import.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { ImportService } from './import.service';
+import { ImportController } from './import.controller';
+
+@Module({
+  controllers: [ImportController],
+  providers: [ImportService]
+})
+export class ImportModule {}

packages/nocodb-nest/src/modules/import/import.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { ImportService } from './import.service';
+
+describe('ImportService', () => {
+  let service: ImportService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [ImportService],
+    }).compile();
+
+    service = module.get<ImportService>(ImportService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/import/import.service.ts

@@ -0,0 +1,4 @@
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class ImportService {}

packages/nocodb-nest/src/modules/plugins/plugins.controller.spec.ts

@@ -0,0 +1,20 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { PluginsController } from './plugins.controller';
+import { PluginsService } from './plugins.service';
+
+describe('PluginsController', () => {
+  let controller: PluginsController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [PluginsController],
+      providers: [PluginsService],
+    }).compile();
+
+    controller = module.get<PluginsController>(PluginsController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/plugins/plugins.controller.ts

@@ -0,0 +1,52 @@
+import { Body, Controller, Get, Param, Patch, Post } from '@nestjs/common';
+import { PagedResponseImpl } from '../../helpers/PagedResponse';
+import { Acl } from '../../middlewares/extract-project-id/extract-project-id.middleware';
+import { PluginsService } from './plugins.service';
+
+// todo: move to a interceptor
+const blockInCloudMw = (_req, res, next) => {
+  if (process.env.NC_CLOUD === 'true') {
+    res.status(403).send('Not allowed');
+  } else next();
+};
+
+@Controller('plugins')
+export class PluginsController {
+  constructor(private readonly pluginsService: PluginsService) {}
+
+  @Get('/api/v1/db/meta/plugins')
+  @Acl('pluginList')
+  async pluginList() {
+    return new PagedResponseImpl(await this.pluginsService.pluginList());
+  }
+
+  @Post('/api/v1/db/meta/plugins/test')
+  @Acl('pluginTest')
+  async pluginTest(@Body() body: any) {
+    return await this.pluginsService.pluginTest({ body: body });
+  }
+
+  @Get('/api/v1/db/meta/plugins/:pluginId')
+  @Acl('pluginRead')
+  async pluginRead(@Param('pluginId') pluginId: string) {
+    return await this.pluginsService.pluginRead({ pluginId: pluginId });
+  }
+
+  @Patch('/api/v1/db/meta/plugins/:pluginId')
+  @Acl('pluginUpdate')
+  async pluginUpdate(@Body() body: any, @Param('pluginId') pluginId: string) {
+    const plugin = await this.pluginsService.pluginUpdate({
+      pluginId: pluginId,
+      plugin: body,
+    });
+    return plugin;
+  }
+
+  @Get('/api/v1/db/meta/plugins/:pluginTitle/status')
+  @Acl('isPluginActive')
+  async isPluginActive(@Param('pluginTitle') pluginTitle: string) {
+    return await this.pluginsService.isPluginActive({
+      pluginTitle: pluginTitle,
+    });
+  }
+}

packages/nocodb-nest/src/modules/plugins/plugins.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { PluginsService } from './plugins.service';
+import { PluginsController } from './plugins.controller';
+
+@Module({
+  controllers: [PluginsController],
+  providers: [PluginsService]
+})
+export class PluginsModule {}

packages/nocodb-nest/src/modules/plugins/plugins.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { PluginsService } from './plugins.service';
+
+describe('PluginsService', () => {
+  let service: PluginsService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [PluginsService],
+    }).compile();
+
+    service = module.get<PluginsService>(PluginsService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/plugins/plugins.service.ts

@@ -0,0 +1,40 @@
+import { Injectable } from '@nestjs/common';
+import { PluginTestReqType, PluginType } from 'nocodb-sdk';
+import { validatePayload } from '../../helpers';
+import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2';
+import { Plugin } from '../../models';
+import { T } from 'nc-help';
+
+@Injectable()
+export class PluginsService {
+  async pluginList() {
+    return await Plugin.list();
+  }
+
+  async pluginTest(param: { body: PluginTestReqType }) {
+    validatePayload(
+      'swagger.json#/components/schemas/PluginTestReq',
+      param.body,
+    );
+
+    T.emit('evt', { evt_type: 'plugin:tested' });
+    return await NcPluginMgrv2.test(param.body);
+  }
+
+  async pluginRead(param: { pluginId: string }) {
+    return await Plugin.get(param.pluginId);
+  }
+  async pluginUpdate(param: { pluginId: string; plugin: PluginType }) {
+    validatePayload('swagger.json#/components/schemas/PluginReq', param.plugin);
+
+    const plugin = await Plugin.update(param.pluginId, param.plugin);
+    T.emit('evt', {
+      evt_type: plugin.active ? 'plugin:installed' : 'plugin:uninstalled',
+      title: plugin.title,
+    });
+    return plugin;
+  }
+  async isPluginActive(param: { pluginTitle: string }) {
+    return await Plugin.isPluginActive(param.pluginTitle);
+  }
+}

packages/nocodb-nest/src/modules/sync/helpers/EntityMap.ts

@@ -0,0 +1,222 @@
+import { Readable } from 'stream';
+import sqlite3 from 'sqlite3';
+
+class EntityMap {
+  initialized: boolean;
+  cols: string[];
+  db: any;
+
+  constructor(...args) {
+    this.initialized = false;
+    this.cols = args.map((arg) => processKey(arg));
+    this.db = new Promise((resolve, reject) => {
+      const db = new sqlite3.Database(':memory:');
+
+      const colStatement =
+        this.cols.length > 0
+          ? this.cols.join(' TEXT, ') + ' TEXT'
+          : 'mappingPlaceholder TEXT';
+      db.run(`CREATE TABLE mapping (${colStatement})`, (err) => {
+        if (err) {
+          console.log(err);
+          reject(err);
+        }
+        resolve(db);
+      });
+    });
+  }
+
+  async init() {
+    if (!this.initialized) {
+      this.db = await this.db;
+      this.initialized = true;
+    }
+  }
+
+  destroy() {
+    if (this.initialized && this.db) {
+      this.db.close();
+    }
+  }
+
+  async addRow(row) {
+    if (!this.initialized) {
+      throw 'Please initialize first!';
+    }
+
+    const cols = Object.keys(row).map((key) => processKey(key));
+    const colStatement = cols.map((key) => `'${key}'`).join(', ');
+    const questionMarks = cols.map(() => '?').join(', ');
+
+    const promises = [];
+
+    for (const col of cols.filter((col) => !this.cols.includes(col))) {
+      promises.push(
+        new Promise((resolve, reject) => {
+          this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => {
+            if (err) {
+              console.log(err);
+              reject(err);
+            }
+            this.cols.push(col);
+            resolve(true);
+          });
+        })
+      );
+    }
+
+    await Promise.all(promises);
+
+    const values = Object.values(row).map((val) => {
+      if (typeof val === 'object') {
+        return `JSON::${JSON.stringify(val)}`;
+      }
+      return val;
+    });
+
+    return new Promise((resolve, reject) => {
+      this.db.run(
+        `INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`,
+        values,
+        (err) => {
+          if (err) {
+            console.log(err);
+            reject(err);
+          }
+          resolve(true);
+        }
+      );
+    });
+  }
+
+  getRow(col, val, res = []): Promise<Record<string, any>> {
+    if (!this.initialized) {
+      throw 'Please initialize first!';
+    }
+    return new Promise((resolve, reject) => {
+      col = processKey(col);
+      res = res.map((r) => processKey(r));
+      this.db.get(
+        `SELECT ${
+          res.length ? res.join(', ') : '*'
+        } FROM mapping WHERE ${col} = ?`,
+        [val],
+        (err, rs) => {
+          if (err) {
+            console.log(err);
+            reject(err);
+          }
+          if (rs) {
+            rs = processResponseRow(rs);
+          }
+          resolve(rs);
+        }
+      );
+    });
+  }
+
+  getCount(): Promise<number> {
+    if (!this.initialized) {
+      throw 'Please initialize first!';
+    }
+    return new Promise((resolve, reject) => {
+      this.db.get(`SELECT COUNT(*) as count FROM mapping`, (err, rs) => {
+        if (err) {
+          console.log(err);
+          reject(err);
+        }
+        resolve(rs.count);
+      });
+    });
+  }
+
+  getStream(res = []): DBStream {
+    if (!this.initialized) {
+      throw 'Please initialize first!';
+    }
+    res = res.map((r) => processKey(r));
+    return new DBStream(
+      this.db,
+      `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping`
+    );
+  }
+
+  getLimit(limit, offset, res = []): Promise<Record<string, any>[]> {
+    if (!this.initialized) {
+      throw 'Please initialize first!';
+    }
+    return new Promise((resolve, reject) => {
+      res = res.map((r) => processKey(r));
+      this.db.all(
+        `SELECT ${
+          res.length ? res.join(', ') : '*'
+        } FROM mapping LIMIT ${limit} OFFSET ${offset}`,
+        (err, rs) => {
+          if (err) {
+            console.log(err);
+            reject(err);
+          }
+          for (let row of rs) {
+            row = processResponseRow(row);
+          }
+          resolve(rs);
+        }
+      );
+    });
+  }
+}
+
+class DBStream extends Readable {
+  db: any;
+  stmt: any;
+  sql: any;
+
+  constructor(db, sql) {
+    super({ objectMode: true });
+    this.db = db;
+    this.sql = sql;
+    this.stmt = this.db.prepare(this.sql);
+    this.on('end', () => this.stmt.finalize());
+  }
+
+  _read() {
+    const stream = this;
+    this.stmt.get(function (err, result) {
+      if (err) {
+        stream.emit('error', err);
+      } else {
+        if (result) {
+          result = processResponseRow(result);
+        }
+        stream.push(result || null);
+      }
+    });
+  }
+}
+
+function processResponseRow(res: any) {
+  for (const key of Object.keys(res)) {
+    if (res[key] && res[key].startsWith('JSON::')) {
+      try {
+        res[key] = JSON.parse(res[key].replace('JSON::', ''));
+      } catch (e) {
+        console.log(e);
+      }
+    }
+    if (revertKey(key) !== key) {
+      res[revertKey(key)] = res[key];
+      delete res[key];
+    }
+  }
+  return res;
+}
+
+function processKey(key) {
+  return key.replace(/'/g, "''").replace(/[A-Z]/g, (match) => `_${match}`);
+}
+
+function revertKey(key) {
+  return key.replace(/''/g, "'").replace(/_[A-Z]/g, (match) => match[1]);
+}
+
+export default EntityMap;

packages/nocodb-nest/src/modules/sync/helpers/NocoSyncDestAdapter.ts

@@ -0,0 +1,6 @@
+export abstract class NocoSyncSourceAdapter {
+  public abstract init(): Promise<void>;
+  public abstract destProjectWrite(): Promise<any>;
+  public abstract destSchemaWrite(): Promise<any>;
+  public abstract destDataWrite(): Promise<any>;
+}

packages/nocodb-nest/src/modules/sync/helpers/NocoSyncSourceAdapter.ts

@@ -0,0 +1,7 @@
+export abstract class NocoSyncSourceAdapter {
+  public abstract init(): Promise<void>;
+  public abstract srcSchemaGet(): Promise<any>;
+  public abstract srcDataLoad(): Promise<any>;
+  public abstract srcDataListen(): Promise<any>;
+  public abstract srcDataPoll(): Promise<any>;
+}

packages/nocodb-nest/src/modules/sync/helpers/fetchAT.ts

@@ -0,0 +1,238 @@
+import axios from 'axios';
+
+const info: any = {
+  initialized: false,
+};
+
+async function initialize(shareId) {
+  info.cookie = '';
+  const url = `https://airtable.com/${shareId}`;
+
+  try {
+    const hreq = await axios
+      .get(url, {
+        headers: {
+          accept:
+            'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+          'accept-language': 'en-US,en;q=0.9',
+          'sec-ch-ua':
+            '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+          'sec-ch-ua-mobile': '?0',
+          'sec-ch-ua-platform': '"Linux"',
+          'sec-fetch-dest': 'document',
+          'sec-fetch-mode': 'navigate',
+          'sec-fetch-site': 'none',
+          'sec-fetch-user': '?1',
+          'upgrade-insecure-requests': '1',
+          'User-Agent':
+            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+        },
+        referrerPolicy: 'strict-origin-when-cross-origin',
+        body: null,
+        method: 'GET',
+      })
+      .then((response) => {
+        for (const ck of response.headers['set-cookie']) {
+          info.cookie += ck.split(';')[0] + '; ';
+        }
+        return response.data;
+      })
+      .catch(() => {
+        throw {
+          message:
+            'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+        };
+      });
+
+    info.headers = JSON.parse(
+      hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim()
+    );
+    info.link = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim());
+    info.baseInfo = decodeURIComponent(info.link)
+      .match(/{(.*)}/g)[0]
+      .split('&')
+      .reduce((result, el) => {
+        try {
+          return Object.assign(
+            result,
+            JSON.parse(el.includes('=') ? el.split('=')[1] : el)
+          );
+        } catch (e) {
+          if (el.includes('=')) {
+            return Object.assign(result, {
+              [el.split('=')[0]]: el.split('=')[1],
+            });
+          }
+        }
+      }, {});
+    info.baseId = info.baseInfo.applicationId;
+    info.initialized = true;
+  } catch (e) {
+    console.log(e);
+    info.initialized = false;
+    if (e.message) {
+      throw e;
+    } else {
+      throw {
+        message:
+          'Error processing Shared Base :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+      };
+    }
+  }
+}
+
+async function read() {
+  if (info.initialized) {
+    const resreq = await axios('https://airtable.com' + info.link, {
+      headers: {
+        accept: '*/*',
+        'accept-language': 'en-US,en;q=0.9',
+        'sec-ch-ua':
+          '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+        'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua-platform': '"Linux"',
+        'sec-fetch-dest': 'empty',
+        'sec-fetch-mode': 'cors',
+        'sec-fetch-site': 'same-origin',
+        'User-Agent':
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+        'x-time-zone': 'Europe/Berlin',
+        cookie: info.cookie,
+        ...info.headers,
+      },
+      referrerPolicy: 'no-referrer',
+      body: null,
+      method: 'GET',
+    })
+      .then((response) => {
+        return response.data;
+      })
+      .catch(() => {
+        throw {
+          message:
+            'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+        };
+      });
+
+    return {
+      schema: resreq.data,
+      baseId: info.baseId,
+      baseInfo: info.baseInfo,
+    };
+  } else {
+    throw {
+      message: 'Error Initializing :: please try again !!',
+    };
+  }
+}
+
+async function readView(viewId) {
+  if (info.initialized) {
+    const resreq = await axios(
+      `https://airtable.com/v0.3/view/${viewId}/readData?` +
+        `stringifiedObjectParams=${encodeURIComponent('{}')}&requestId=${
+          info.baseInfo.requestId
+        }&accessPolicy=${encodeURIComponent(
+          JSON.stringify({
+            allowedActions: info.baseInfo.allowedActions,
+            shareId: info.baseInfo.shareId,
+            applicationId: info.baseInfo.applicationId,
+            generationNumber: info.baseInfo.generationNumber,
+            expires: info.baseInfo.expires,
+            signature: info.baseInfo.signature,
+          })
+        )}`,
+      {
+        headers: {
+          accept: '*/*',
+          'accept-language': 'en-US,en;q=0.9',
+          'sec-ch-ua':
+            '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+          'sec-ch-ua-mobile': '?0',
+          'sec-ch-ua-platform': '"Linux"',
+          'sec-fetch-dest': 'empty',
+          'sec-fetch-mode': 'cors',
+          'sec-fetch-site': 'same-origin',
+          'User-Agent':
+            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+          'x-time-zone': 'Europe/Berlin',
+          cookie: info.cookie,
+          ...info.headers,
+        },
+        referrerPolicy: 'no-referrer',
+        body: null,
+        method: 'GET',
+      }
+    )
+      .then((response) => {
+        return response.data;
+      })
+      .catch(() => {
+        throw {
+          message:
+            'Error Reading View :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+        };
+      });
+    return { view: resreq.data };
+  } else {
+    throw {
+      message: 'Error Initializing :: please try again !!',
+    };
+  }
+}
+
+async function readTemplate(templateId) {
+  if (!info.initialized) {
+    await initialize('shrO8aYf3ybwSdDKn');
+  }
+  const resreq = await axios(
+    `https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
+    {
+      headers: {
+        accept: '*/*',
+        'accept-language': 'en-US,en;q=0.9',
+        'sec-ch-ua':
+          '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+        'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua-platform': '"Linux"',
+        'sec-fetch-dest': 'empty',
+        'sec-fetch-mode': 'cors',
+        'sec-fetch-site': 'same-origin',
+        'User-Agent':
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+        'x-time-zone': 'Europe/Berlin',
+        cookie: info.cookie,
+        ...info.headers,
+      },
+      referrer: 'https://www.airtable.com/',
+      referrerPolicy: 'same-origin',
+      body: null,
+      method: 'GET',
+      mode: 'cors',
+      credentials: 'include',
+    }
+  )
+    .then((response) => {
+      return response.data;
+    })
+    .catch(() => {
+      throw {
+        message:
+          'Error Fetching :: Ensure www.airtable.com/templates/featured/<TemplateID> is accessible.',
+      };
+    });
+  return { template: resreq };
+}
+
+function unicodeToChar(text) {
+  return text.replace(/\\u[\dA-F]{4}/gi, function (match) {
+    return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16));
+  });
+}
+
+export default {
+  initialize,
+  read,
+  readView,
+  readTemplate,
+};

packages/nocodb-nest/src/modules/sync/helpers/readAndProcessData.ts

@@ -0,0 +1,346 @@
+import { RelationTypes, UITypes } from 'nocodb-sdk';
+import { bulkDataService, tableService } from '../..';
+import EntityMap from './EntityMap';
+import type { AirtableBase } from 'airtable/lib/airtable_base';
+import type { TableType } from 'nocodb-sdk';
+
+const BULK_DATA_BATCH_SIZE = 500;
+const ASSOC_BULK_DATA_BATCH_SIZE = 1000;
+const BULK_PARALLEL_PROCESS = 5;
+
+async function readAllData({
+  table,
+  fields,
+  base,
+  logBasic = (_str) => {},
+}: {
+  table: { title?: string };
+  fields?;
+  base: AirtableBase;
+  logBasic?: (string) => void;
+  logDetailed?: (string) => void;
+}): Promise<EntityMap> {
+  return new Promise((resolve, reject) => {
+    let data = null;
+
+    const selectParams: any = {
+      pageSize: 100,
+    };
+
+    if (fields) selectParams.fields = fields;
+
+    base(table.title)
+      .select(selectParams)
+      .eachPage(
+        async function page(records, fetchNextPage) {
+          if (!data) {
+            data = new EntityMap();
+            await data.init();
+          }
+
+          for await (const record of records) {
+            await data.addRow({ id: record.id, ...record.fields });
+          }
+
+          const tmpLength = await data.getCount();
+
+          logBasic(
+            `:: Reading '${table.title}' data :: ${Math.max(
+              1,
+              tmpLength - records.length,
+            )} - ${tmpLength}`,
+          );
+
+          // To fetch the next page of records, call `fetchNextPage`.
+          // If there are more records, `page` will get called again.
+          // If there are no more records, `done` will get called.
+          fetchNextPage();
+        },
+        async function done(err) {
+          if (err) {
+            console.error(err);
+            return reject(err);
+          }
+          resolve(data);
+        },
+      );
+  });
+}
+
+export async function importData({
+  projectName,
+  table,
+  base,
+  nocoBaseDataProcessing_v2,
+  sDB,
+  logDetailed = (_str) => {},
+  logBasic = (_str) => {},
+}: {
+  projectName: string;
+  table: { title?: string; id?: string };
+  fields?;
+  base: AirtableBase;
+  logBasic: (string) => void;
+  logDetailed: (string) => void;
+  nocoBaseDataProcessing_v2;
+  sDB;
+}): Promise<EntityMap> {
+  try {
+    // @ts-ignore
+    const records = await readAllData({
+      table,
+      base,
+      logDetailed,
+      logBasic,
+    });
+
+    await new Promise(async (resolve) => {
+      const readable = records.getStream();
+      const allRecordsCount = await records.getCount();
+      const promises = [];
+      let tempData = [];
+      let importedCount = 0;
+      let activeProcess = 0;
+      readable.on('data', async (record) => {
+        promises.push(
+          new Promise(async (resolve) => {
+            activeProcess++;
+            if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause();
+            const { id: rid, ...fields } = record;
+            const r = await nocoBaseDataProcessing_v2(sDB, table, {
+              id: rid,
+              fields,
+            });
+            tempData.push(r);
+
+            if (tempData.length >= BULK_DATA_BATCH_SIZE) {
+              let insertArray = tempData.splice(0, tempData.length);
+
+              await bulkDataService.bulkDataInsert({
+                projectName,
+                tableName: table.title,
+                body: insertArray,
+                cookie: {},
+              });
+
+              logBasic(
+                `:: Importing '${
+                  table.title
+                }' data :: ${importedCount} - ${Math.min(
+                  importedCount + BULK_DATA_BATCH_SIZE,
+                  allRecordsCount,
+                )}`,
+              );
+              importedCount += insertArray.length;
+              insertArray = [];
+            }
+            activeProcess--;
+            if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume();
+            resolve(true);
+          }),
+        );
+      });
+      readable.on('end', async () => {
+        await Promise.all(promises);
+        if (tempData.length > 0) {
+          await bulkDataService.bulkDataInsert({
+            projectName,
+            tableName: table.title,
+            body: tempData,
+            cookie: {},
+          });
+
+          logBasic(
+            `:: Importing '${
+              table.title
+            }' data :: ${importedCount} - ${Math.min(
+              importedCount + BULK_DATA_BATCH_SIZE,
+              allRecordsCount,
+            )}`,
+          );
+          importedCount += tempData.length;
+          tempData = [];
+        }
+        resolve(true);
+      });
+    });
+
+    return records;
+  } catch (e) {
+    console.log(e);
+    return null;
+  }
+}
+
+export async function importLTARData({
+  table,
+  fields,
+  base,
+  projectName,
+  insertedAssocRef = {},
+  logDetailed = (_str) => {},
+  logBasic = (_str) => {},
+  records,
+  atNcAliasRef,
+  ncLinkMappingTable,
+  syncDB,
+}: {
+  projectName: string;
+  table: { title?: string; id?: string };
+  fields;
+  base: AirtableBase;
+  logDetailed: (string) => void;
+  logBasic: (string) => void;
+  insertedAssocRef: { [assocTableId: string]: boolean };
+  records?: EntityMap;
+  atNcAliasRef: {
+    [ncTableId: string]: {
+      [ncTitle: string]: string;
+    };
+  };
+  ncLinkMappingTable: Record<string, Record<string, any>>[];
+  syncDB;
+}) {
+  const assocTableMetas: Array<{
+    modelMeta: { id?: string; title?: string };
+    colMeta: { title?: string };
+    curCol: { title?: string };
+    refCol: { title?: string };
+  }> = [];
+  const allData =
+    records ||
+    (await readAllData({
+      table,
+      fields,
+      base,
+      logDetailed,
+      logBasic,
+    }));
+
+  const modelMeta: any = await tableService.getTableWithAccessibleViews({
+    tableId: table.id,
+    user: syncDB.user,
+  });
+
+  for (const colMeta of modelMeta.columns) {
+    // skip columns which are not LTAR and Many to many
+    if (
+      colMeta.uidt !== UITypes.LinkToAnotherRecord ||
+      colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY
+    ) {
+      continue;
+    }
+
+    // skip if already inserted
+    if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue;
+
+    // self links: skip if the column under consideration is the add-on column NocoDB creates
+    if (ncLinkMappingTable.every((a) => a.nc.title !== colMeta.title)) continue;
+
+    // mark as inserted
+    insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true;
+
+    const assocModelMeta: TableType =
+      (await tableService.getTableWithAccessibleViews({
+        tableId: colMeta.colOptions.fk_mm_model_id,
+        user: syncDB.user,
+      })) as any;
+
+    // extract associative table and columns meta
+    assocTableMetas.push({
+      modelMeta: assocModelMeta,
+      colMeta,
+      curCol: assocModelMeta.columns.find(
+        (c) => c.id === colMeta.colOptions.fk_mm_child_column_id,
+      ),
+      refCol: assocModelMeta.columns.find(
+        (c) => c.id === colMeta.colOptions.fk_mm_parent_column_id,
+      ),
+    });
+  }
+
+  let nestedLinkCnt = 0;
+  // Iterate over all related M2M associative  table
+  for await (const assocMeta of assocTableMetas) {
+    let assocTableData = [];
+    let importedCount = 0;
+
+    //  extract insert data from records
+    await new Promise((resolve) => {
+      const promises = [];
+      const readable = allData.getStream();
+      let activeProcess = 0;
+      readable.on('data', async (record) => {
+        promises.push(
+          new Promise(async (resolve) => {
+            activeProcess++;
+            if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause();
+            const { id: _atId, ...rec } = record;
+
+            // todo: use actual alias instead of sanitized
+            assocTableData.push(
+              ...(
+                rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || []
+              ).map((id) => ({
+                [assocMeta.curCol.title]: record.id,
+                [assocMeta.refCol.title]: id,
+              })),
+            );
+
+            if (assocTableData.length >= ASSOC_BULK_DATA_BATCH_SIZE) {
+              let insertArray = assocTableData.splice(0, assocTableData.length);
+              logBasic(
+                `:: Importing '${
+                  table.title
+                }' LTAR data :: ${importedCount} - ${Math.min(
+                  importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
+                  insertArray.length,
+                )}`,
+              );
+
+              await bulkDataService.bulkDataInsert({
+                projectName,
+                tableName: assocMeta.modelMeta.title,
+                body: insertArray,
+                cookie: {},
+              });
+
+              importedCount += insertArray.length;
+              insertArray = [];
+            }
+            activeProcess--;
+            if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume();
+            resolve(true);
+          }),
+        );
+      });
+      readable.on('end', async () => {
+        await Promise.all(promises);
+        if (assocTableData.length >= 0) {
+          logBasic(
+            `:: Importing '${
+              table.title
+            }' LTAR data :: ${importedCount} - ${Math.min(
+              importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
+              assocTableData.length,
+            )}`,
+          );
+
+          await bulkDataService.bulkDataInsert({
+            projectName,
+            tableName: assocMeta.modelMeta.title,
+            body: assocTableData,
+            cookie: {},
+          });
+
+          importedCount += assocTableData.length;
+          assocTableData = [];
+        }
+        resolve(true);
+      });
+    });
+
+    nestedLinkCnt += importedCount;
+  }
+  return nestedLinkCnt;
+}

packages/nocodb-nest/src/modules/sync/helpers/syncMap.ts

@@ -0,0 +1,31 @@
+export const mapTbl = {};
+
+// static mapping records between aTblId && ncId
+export const addToMappingTbl = function addToMappingTbl(
+  aTblId,
+  ncId,
+  ncName,
+  parent?
+) {
+  mapTbl[aTblId] = {
+    ncId: ncId,
+    ncParent: parent,
+    // name added to assist in quick debug
+    ncName: ncName,
+  };
+};
+
+// get NcID from airtable ID
+export const getNcIdFromAtId = function getNcIdFromAtId(aId) {
+  return mapTbl[aId]?.ncId;
+};
+
+// get nc Parent from airtable ID
+export const getNcParentFromAtId = function getNcParentFromAtId(aId) {
+  return mapTbl[aId]?.ncParent;
+};
+
+// get nc-title from airtable ID
+export const getNcNameFromAtId = function getNcNameFromAtId(aId) {
+  return mapTbl[aId]?.ncName;
+};

packages/nocodb-nest/src/modules/sync/sync.controller.spec.ts

@@ -0,0 +1,20 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { SyncController } from './sync.controller';
+import { SyncService } from './sync.service';
+
+describe('SyncController', () => {
+  let controller: SyncController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [SyncController],
+      providers: [SyncService],
+    }).compile();
+
+    controller = module.get<SyncController>(SyncController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/sync/sync.controller.ts

@@ -0,0 +1,67 @@
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  Param,
+  Patch,
+  Post,
+  Req,
+} from '@nestjs/common';
+import { Acl } from '../../middlewares/extract-project-id/extract-project-id.middleware';
+import { SyncService } from './sync.service';
+
+@Controller('sync')
+export class SyncController {
+  constructor(private readonly syncService: SyncService) {}
+
+  @Get([
+    '/api/v1/db/meta/projects/:projectId/syncs',
+    '/api/v1/db/meta/projects/:projectId/syncs/:baseId',
+  ])
+  @Acl('syncSourceList')
+  async syncSourceList(
+    @Param('projectId') projectId: string,
+    @Param('baseId') baseId?: string,
+  ) {
+    return await this.syncService.syncSourceList({
+      projectId,
+      baseId,
+    });
+  }
+
+  @Post([
+    '/api/v1/db/meta/projects/:projectId/syncs',
+    '/api/v1/db/meta/projects/:projectId/syncs/:baseId',
+  ])
+  @Acl('syncSourceCreate')
+  async syncCreate(
+    @Param('projectId') projectId: string,
+    @Body() body: any,
+    @Req() req,
+    @Param('baseId') baseId?: string,
+  ) {
+    return await this.syncService.syncCreate({
+      projectId: projectId,
+      baseId: baseId,
+      userId: (req as any).user.id,
+      syncPayload: body,
+    });
+  }
+
+  @Delete('/api/v1/db/meta/syncs/:syncId')
+  @Acl('syncSourceDelete')
+  async syncDelete(@Param('syncId') syncId: string) {
+    return await this.syncService.syncDelete({
+      syncId: syncId,
+    });
+  }
+
+  @Patch('/api/v1/db/meta/syncs/:syncId')
+  async syncUpdate(@Param('syncId') syncId: string, @Body() body: any) {
+    return await this.syncService.syncUpdate({
+      syncId: syncId,
+      syncPayload: body,
+    });
+  }
+}

packages/nocodb-nest/src/modules/sync/sync.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { SyncService } from './sync.service';
+import { SyncController } from './sync.controller';
+
+@Module({
+  controllers: [SyncController],
+  providers: [SyncService]
+})
+export class SyncModule {}

packages/nocodb-nest/src/modules/sync/sync.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { SyncService } from './sync.service';
+
+describe('SyncService', () => {
+  let service: SyncService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [SyncService],
+    }).compile();
+
+    service = module.get<SyncService>(SyncService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/sync/sync.service.ts

@@ -0,0 +1,45 @@
+import { Injectable } from '@nestjs/common';
+import { PagedResponseImpl } from '../../helpers/PagedResponse';
+import { Project, SyncSource } from '../../models';
+import { T } from 'nc-help';
+
+@Injectable()
+export class SyncService {
+  async syncSourceList(param: { projectId: string; baseId?: string }) {
+    return new PagedResponseImpl(
+      await SyncSource.list(param.projectId, param.baseId),
+    );
+  }
+
+  async syncCreate(param: {
+    projectId: string;
+    baseId?: string;
+    userId: string;
+    syncPayload: Partial<SyncSource>;
+  }) {
+    T.emit('evt', { evt_type: 'syncSource:created' });
+    const project = await Project.getWithInfo(param.projectId);
+
+    const sync = await SyncSource.insert({
+      ...param.syncPayload,
+      fk_user_id: param.userId,
+      base_id: param.baseId ? param.baseId : project.bases[0].id,
+      project_id: param.projectId,
+    });
+    return sync;
+  }
+
+  async syncDelete(param: { syncId: string }) {
+    T.emit('evt', { evt_type: 'syncSource:deleted' });
+    return await SyncSource.delete(param.syncId);
+  }
+
+  async syncUpdate(param: {
+    syncId: string;
+    syncPayload: Partial<SyncSource>;
+  }) {
+    T.emit('evt', { evt_type: 'syncSource:updated' });
+
+    return await SyncSource.update(param.syncId, param.syncPayload);
+  }
+}

packages/nocodb-nest/src/modules/test/test.controller.spec.ts

@@ -0,0 +1,20 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { TestController } from './test.controller';
+import { TestService } from './test.service';
+
+describe('TestController', () => {
+  let controller: TestController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [TestController],
+      providers: [TestService],
+    }).compile();
+
+    controller = module.get<TestController>(TestController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/test/test.controller.ts

@@ -0,0 +1,7 @@
+import { Controller } from '@nestjs/common';
+import { TestService } from './test.service';
+
+@Controller('test')
+export class TestController {
+  constructor(private readonly testService: TestService) {}
+}

packages/nocodb-nest/src/modules/test/test.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { TestService } from './test.service';
+import { TestController } from './test.controller';
+
+@Module({
+  controllers: [TestController],
+  providers: [TestService]
+})
+export class TestModule {}

packages/nocodb-nest/src/modules/test/test.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { TestService } from './test.service';
+
+describe('TestService', () => {
+  let service: TestService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [TestService],
+    }).compile();
+
+    service = module.get<TestService>(TestService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/test/test.service.ts

@@ -0,0 +1,4 @@
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class TestService {}