From 34a9fe5b81ca7cd3aea136339fb2217f09578012 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 10 Apr 2023 01:25:46 +0530 Subject: [PATCH] feat: plugi, caches apis Signed-off-by: Pranav C --- packages/nocodb-nest/src/app.module.ts | 7 +- .../modules/caches/caches.controller.spec.ts | 20 + .../src/modules/caches/caches.controller.ts | 7 + .../src/modules/caches/caches.module.ts | 9 + .../src/modules/caches/caches.service.spec.ts | 18 + .../src/modules/caches/caches.service.ts | 4 + .../modules/import/import.controller.spec.ts | 20 + .../src/modules/import/import.controller.ts | 142 + .../src/modules/import/import.module.ts | 9 + .../src/modules/import/import.service.spec.ts | 18 + .../src/modules/import/import.service.ts | 4 + .../plugins/plugins.controller.spec.ts | 20 + .../src/modules/plugins/plugins.controller.ts | 52 + .../src/modules/plugins/plugins.module.ts | 9 + .../modules/plugins/plugins.service.spec.ts | 18 + .../src/modules/plugins/plugins.service.ts | 40 + .../src/modules/sync/helpers/EntityMap.ts | 222 ++ .../sync/helpers/NocoSyncDestAdapter.ts | 6 + .../sync/helpers/NocoSyncSourceAdapter.ts | 7 + .../src/modules/sync/helpers/fetchAT.ts | 238 ++ .../src/modules/sync/helpers/job.ts | 2444 +++++++++++++++++ .../sync/helpers/readAndProcessData.ts | 346 +++ .../src/modules/sync/helpers/syncMap.ts | 31 + .../src/modules/sync/sync.controller.spec.ts | 20 + .../src/modules/sync/sync.controller.ts | 67 + .../src/modules/sync/sync.module.ts | 9 + .../src/modules/sync/sync.service.spec.ts | 18 + .../src/modules/sync/sync.service.ts | 45 + .../src/modules/test/test.controller.spec.ts | 20 + .../src/modules/test/test.controller.ts | 7 + .../src/modules/test/test.module.ts | 9 + .../src/modules/test/test.service.spec.ts | 18 + .../src/modules/test/test.service.ts | 4 + 33 files changed, 3907 insertions(+), 1 deletion(-) create mode 100644 packages/nocodb-nest/src/modules/caches/caches.controller.spec.ts create mode 100644 packages/nocodb-nest/src/modules/caches/caches.controller.ts create mode 100644 packages/nocodb-nest/src/modules/caches/caches.module.ts create mode 100644 packages/nocodb-nest/src/modules/caches/caches.service.spec.ts create mode 100644 packages/nocodb-nest/src/modules/caches/caches.service.ts create mode 100644 packages/nocodb-nest/src/modules/import/import.controller.spec.ts create mode 100644 packages/nocodb-nest/src/modules/import/import.controller.ts create mode 100644 packages/nocodb-nest/src/modules/import/import.module.ts create mode 100644 packages/nocodb-nest/src/modules/import/import.service.spec.ts create mode 100644 packages/nocodb-nest/src/modules/import/import.service.ts create mode 100644 packages/nocodb-nest/src/modules/plugins/plugins.controller.spec.ts create mode 100644 packages/nocodb-nest/src/modules/plugins/plugins.controller.ts create mode 100644 packages/nocodb-nest/src/modules/plugins/plugins.module.ts create mode 100644 packages/nocodb-nest/src/modules/plugins/plugins.service.spec.ts create mode 100644 packages/nocodb-nest/src/modules/plugins/plugins.service.ts create mode 100644 packages/nocodb-nest/src/modules/sync/helpers/EntityMap.ts create mode 100644 packages/nocodb-nest/src/modules/sync/helpers/NocoSyncDestAdapter.ts create mode 100644 packages/nocodb-nest/src/modules/sync/helpers/NocoSyncSourceAdapter.ts create mode 100644 packages/nocodb-nest/src/modules/sync/helpers/fetchAT.ts create mode 100644 packages/nocodb-nest/src/modules/sync/helpers/job.ts create mode 100644 packages/nocodb-nest/src/modules/sync/helpers/readAndProcessData.ts create mode 100644 packages/nocodb-nest/src/modules/sync/helpers/syncMap.ts create mode 100644 packages/nocodb-nest/src/modules/sync/sync.controller.spec.ts create mode 100644 packages/nocodb-nest/src/modules/sync/sync.controller.ts create mode 100644 packages/nocodb-nest/src/modules/sync/sync.module.ts create mode 100644 packages/nocodb-nest/src/modules/sync/sync.service.spec.ts create mode 100644 packages/nocodb-nest/src/modules/sync/sync.service.ts create mode 100644 packages/nocodb-nest/src/modules/test/test.controller.spec.ts create mode 100644 packages/nocodb-nest/src/modules/test/test.controller.ts create mode 100644 packages/nocodb-nest/src/modules/test/test.module.ts create mode 100644 packages/nocodb-nest/src/modules/test/test.service.spec.ts create mode 100644 packages/nocodb-nest/src/modules/test/test.service.ts diff --git a/packages/nocodb-nest/src/app.module.ts b/packages/nocodb-nest/src/app.module.ts index 419bd32804..bb7b75e2c7 100644 --- a/packages/nocodb-nest/src/app.module.ts +++ b/packages/nocodb-nest/src/app.module.ts @@ -41,9 +41,14 @@ import { ApiDocsModule } from './modules/api-docs/api-docs.module'; import { PublicMetasModule } from './modules/public-metas/public-metas.module'; import { PublicDatasModule } from './modules/public-datas/public-datas.module'; import { PublicDatasExportModule } from './modules/public-datas-export/public-datas-export.module'; +import { SyncModule } from './modules/sync/sync.module'; +import { ImportModule } from './modules/import/import.module'; +import { CachesModule } from './modules/caches/caches.module'; +import { TestModule } from './modules/test/test.module'; +import { PluginsModule } from './modules/plugins/plugins.module'; @Module({ - imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule, AuditsModule, DatasModule, DataAliasModule, ApiDocsModule, PublicMetasModule, PublicDatasModule, PublicDatasExportModule], + imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule, AuditsModule, DatasModule, DataAliasModule, ApiDocsModule, PublicMetasModule, PublicDatasModule, PublicDatasExportModule, SyncModule, ImportModule, CachesModule, TestModule, PluginsModule], controllers: [], providers: [Connection, MetaService, JwtStrategy, ExtractProjectIdMiddleware], exports: [Connection, MetaService], diff --git a/packages/nocodb-nest/src/modules/caches/caches.controller.spec.ts b/packages/nocodb-nest/src/modules/caches/caches.controller.spec.ts new file mode 100644 index 0000000000..185c027f22 --- /dev/null +++ b/packages/nocodb-nest/src/modules/caches/caches.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { CachesController } from './caches.controller'; +import { CachesService } from './caches.service'; + +describe('CachesController', () => { + let controller: CachesController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [CachesController], + providers: [CachesService], + }).compile(); + + controller = module.get(CachesController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/caches/caches.controller.ts b/packages/nocodb-nest/src/modules/caches/caches.controller.ts new file mode 100644 index 0000000000..471e2b53b0 --- /dev/null +++ b/packages/nocodb-nest/src/modules/caches/caches.controller.ts @@ -0,0 +1,7 @@ +import { Controller } from '@nestjs/common'; +import { CachesService } from './caches.service'; + +@Controller('caches') +export class CachesController { + constructor(private readonly cachesService: CachesService) {} +} diff --git a/packages/nocodb-nest/src/modules/caches/caches.module.ts b/packages/nocodb-nest/src/modules/caches/caches.module.ts new file mode 100644 index 0000000000..cd84c55d8a --- /dev/null +++ b/packages/nocodb-nest/src/modules/caches/caches.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { CachesService } from './caches.service'; +import { CachesController } from './caches.controller'; + +@Module({ + controllers: [CachesController], + providers: [CachesService] +}) +export class CachesModule {} diff --git a/packages/nocodb-nest/src/modules/caches/caches.service.spec.ts b/packages/nocodb-nest/src/modules/caches/caches.service.spec.ts new file mode 100644 index 0000000000..d5398c7bfa --- /dev/null +++ b/packages/nocodb-nest/src/modules/caches/caches.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { CachesService } from './caches.service'; + +describe('CachesService', () => { + let service: CachesService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [CachesService], + }).compile(); + + service = module.get(CachesService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/caches/caches.service.ts b/packages/nocodb-nest/src/modules/caches/caches.service.ts new file mode 100644 index 0000000000..4b3dab6de5 --- /dev/null +++ b/packages/nocodb-nest/src/modules/caches/caches.service.ts @@ -0,0 +1,4 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class CachesService {} diff --git a/packages/nocodb-nest/src/modules/import/import.controller.spec.ts b/packages/nocodb-nest/src/modules/import/import.controller.spec.ts new file mode 100644 index 0000000000..85db7c3b05 --- /dev/null +++ b/packages/nocodb-nest/src/modules/import/import.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ImportController } from './import.controller'; +import { ImportService } from './import.service'; + +describe('ImportController', () => { + let controller: ImportController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [ImportController], + providers: [ImportService], + }).compile(); + + controller = module.get(ImportController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/import/import.controller.ts b/packages/nocodb-nest/src/modules/import/import.controller.ts new file mode 100644 index 0000000000..2117020580 --- /dev/null +++ b/packages/nocodb-nest/src/modules/import/import.controller.ts @@ -0,0 +1,142 @@ +import { Controller } from '@nestjs/common'; +import { ImportService } from './import.service'; + +const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB'; +const AIRTABLE_PROGRESS_JOB = 'AIRTABLE_PROGRESS_JOB'; + +enum SyncStatus { + PROGRESS = 'PROGRESS', + COMPLETED = 'COMPLETED', + FAILED = 'FAILED', +} + +@Controller('import') +export class ImportController { + constructor(private readonly importService: ImportService) {} + + + + + default ( + router: Router, + sv: Server, + jobs: { [id: string]: { last_message: any } } +) => { + // add importer job handler and progress notification job handler + NocoJobs.jobsMgr.addJobWorker( + AIRTABLE_IMPORT_JOB, + syncService.airtableImportJob + ); + NocoJobs.jobsMgr.addJobWorker( + AIRTABLE_PROGRESS_JOB, + ({ payload, progress }) => { + sv.to(payload?.id).emit('progress', { + msg: progress?.msg, + level: progress?.level, + status: progress?.status, + }); + + if (payload?.id in jobs) { + jobs[payload?.id].last_message = { + msg: progress?.msg, + level: progress?.level, + status: progress?.status, + }; + } + } + ); + + NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => { + NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { + payload, + progress: { + msg: progress?.msg, + level: progress?.level, + status: progress?.status, + }, + }); + }); + NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, (payload) => { + NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { + payload, + progress: { + msg: 'Complete!', + status: SyncStatus.COMPLETED, + }, + }); + delete jobs[payload?.id]; + }); + NocoJobs.jobsMgr.addFailureCbk(AIRTABLE_IMPORT_JOB, (payload, error: any) => { + NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { + payload, + progress: { + msg: error?.message || 'Failed due to some internal error', + status: SyncStatus.FAILED, + }, + }); + delete jobs[payload?.id]; + }); + + router.post( + '/api/v1/db/meta/import/airtable', + catchError((req, res) => { + NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { + id: req.query.id, + ...req.body, + }); + res.json({}); + }) + ); + router.post( + '/api/v1/db/meta/syncs/:syncId/trigger', + catchError(async (req: Request, res) => { + if (req.params.syncId in jobs) { + NcError.badRequest('Sync already in progress'); + } + + const syncSource = await SyncSource.get(req.params.syncId); + + const user = await syncSource.getUser(); + const token = userService.genJwt(user, Noco.getConfig()); + + // Treat default baseUrl as siteUrl from req object + let baseURL = (req as any).ncSiteUrl; + + // if environment value avail use it + // or if it's docker construct using `PORT` + if (process.env.NC_DOCKER) { + baseURL = `http://localhost:${process.env.PORT || 8080}`; + } + + setTimeout(() => { + NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { + id: req.params.syncId, + ...(syncSource?.details || {}), + projectId: syncSource.project_id, + baseId: syncSource.base_id, + authToken: token, + baseURL, + user: user, + }); + }, 1000); + + jobs[req.params.syncId] = { + last_message: { + msg: 'Sync started', + }, + }; + res.json({}); + }) + ); + router.post( + '/api/v1/db/meta/syncs/:syncId/abort', + catchError(async (req: Request, res) => { + if (req.params.syncId in jobs) { + delete jobs[req.params.syncId]; + } + res.json({}); + }) + ); +}; + +} diff --git a/packages/nocodb-nest/src/modules/import/import.module.ts b/packages/nocodb-nest/src/modules/import/import.module.ts new file mode 100644 index 0000000000..30f59328fc --- /dev/null +++ b/packages/nocodb-nest/src/modules/import/import.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { ImportService } from './import.service'; +import { ImportController } from './import.controller'; + +@Module({ + controllers: [ImportController], + providers: [ImportService] +}) +export class ImportModule {} diff --git a/packages/nocodb-nest/src/modules/import/import.service.spec.ts b/packages/nocodb-nest/src/modules/import/import.service.spec.ts new file mode 100644 index 0000000000..96ac7b2e4c --- /dev/null +++ b/packages/nocodb-nest/src/modules/import/import.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ImportService } from './import.service'; + +describe('ImportService', () => { + let service: ImportService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ImportService], + }).compile(); + + service = module.get(ImportService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/import/import.service.ts b/packages/nocodb-nest/src/modules/import/import.service.ts new file mode 100644 index 0000000000..dc6bac1141 --- /dev/null +++ b/packages/nocodb-nest/src/modules/import/import.service.ts @@ -0,0 +1,4 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class ImportService {} diff --git a/packages/nocodb-nest/src/modules/plugins/plugins.controller.spec.ts b/packages/nocodb-nest/src/modules/plugins/plugins.controller.spec.ts new file mode 100644 index 0000000000..139f2bdf39 --- /dev/null +++ b/packages/nocodb-nest/src/modules/plugins/plugins.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { PluginsController } from './plugins.controller'; +import { PluginsService } from './plugins.service'; + +describe('PluginsController', () => { + let controller: PluginsController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [PluginsController], + providers: [PluginsService], + }).compile(); + + controller = module.get(PluginsController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/plugins/plugins.controller.ts b/packages/nocodb-nest/src/modules/plugins/plugins.controller.ts new file mode 100644 index 0000000000..b96b7f2ddd --- /dev/null +++ b/packages/nocodb-nest/src/modules/plugins/plugins.controller.ts @@ -0,0 +1,52 @@ +import { Body, Controller, Get, Param, Patch, Post } from '@nestjs/common'; +import { PagedResponseImpl } from '../../helpers/PagedResponse'; +import { Acl } from '../../middlewares/extract-project-id/extract-project-id.middleware'; +import { PluginsService } from './plugins.service'; + +// todo: move to a interceptor +const blockInCloudMw = (_req, res, next) => { + if (process.env.NC_CLOUD === 'true') { + res.status(403).send('Not allowed'); + } else next(); +}; + +@Controller('plugins') +export class PluginsController { + constructor(private readonly pluginsService: PluginsService) {} + + @Get('/api/v1/db/meta/plugins') + @Acl('pluginList') + async pluginList() { + return new PagedResponseImpl(await this.pluginsService.pluginList()); + } + + @Post('/api/v1/db/meta/plugins/test') + @Acl('pluginTest') + async pluginTest(@Body() body: any) { + return await this.pluginsService.pluginTest({ body: body }); + } + + @Get('/api/v1/db/meta/plugins/:pluginId') + @Acl('pluginRead') + async pluginRead(@Param('pluginId') pluginId: string) { + return await this.pluginsService.pluginRead({ pluginId: pluginId }); + } + + @Patch('/api/v1/db/meta/plugins/:pluginId') + @Acl('pluginUpdate') + async pluginUpdate(@Body() body: any, @Param('pluginId') pluginId: string) { + const plugin = await this.pluginsService.pluginUpdate({ + pluginId: pluginId, + plugin: body, + }); + return plugin; + } + + @Get('/api/v1/db/meta/plugins/:pluginTitle/status') + @Acl('isPluginActive') + async isPluginActive(@Param('pluginTitle') pluginTitle: string) { + return await this.pluginsService.isPluginActive({ + pluginTitle: pluginTitle, + }); + } +} diff --git a/packages/nocodb-nest/src/modules/plugins/plugins.module.ts b/packages/nocodb-nest/src/modules/plugins/plugins.module.ts new file mode 100644 index 0000000000..3e12ebaf21 --- /dev/null +++ b/packages/nocodb-nest/src/modules/plugins/plugins.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { PluginsService } from './plugins.service'; +import { PluginsController } from './plugins.controller'; + +@Module({ + controllers: [PluginsController], + providers: [PluginsService] +}) +export class PluginsModule {} diff --git a/packages/nocodb-nest/src/modules/plugins/plugins.service.spec.ts b/packages/nocodb-nest/src/modules/plugins/plugins.service.spec.ts new file mode 100644 index 0000000000..a6ffe5c5c1 --- /dev/null +++ b/packages/nocodb-nest/src/modules/plugins/plugins.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { PluginsService } from './plugins.service'; + +describe('PluginsService', () => { + let service: PluginsService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [PluginsService], + }).compile(); + + service = module.get(PluginsService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/plugins/plugins.service.ts b/packages/nocodb-nest/src/modules/plugins/plugins.service.ts new file mode 100644 index 0000000000..98d6bc4937 --- /dev/null +++ b/packages/nocodb-nest/src/modules/plugins/plugins.service.ts @@ -0,0 +1,40 @@ +import { Injectable } from '@nestjs/common'; +import { PluginTestReqType, PluginType } from 'nocodb-sdk'; +import { validatePayload } from '../../helpers'; +import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2'; +import { Plugin } from '../../models'; +import { T } from 'nc-help'; + +@Injectable() +export class PluginsService { + async pluginList() { + return await Plugin.list(); + } + + async pluginTest(param: { body: PluginTestReqType }) { + validatePayload( + 'swagger.json#/components/schemas/PluginTestReq', + param.body, + ); + + T.emit('evt', { evt_type: 'plugin:tested' }); + return await NcPluginMgrv2.test(param.body); + } + + async pluginRead(param: { pluginId: string }) { + return await Plugin.get(param.pluginId); + } + async pluginUpdate(param: { pluginId: string; plugin: PluginType }) { + validatePayload('swagger.json#/components/schemas/PluginReq', param.plugin); + + const plugin = await Plugin.update(param.pluginId, param.plugin); + T.emit('evt', { + evt_type: plugin.active ? 'plugin:installed' : 'plugin:uninstalled', + title: plugin.title, + }); + return plugin; + } + async isPluginActive(param: { pluginTitle: string }) { + return await Plugin.isPluginActive(param.pluginTitle); + } +} diff --git a/packages/nocodb-nest/src/modules/sync/helpers/EntityMap.ts b/packages/nocodb-nest/src/modules/sync/helpers/EntityMap.ts new file mode 100644 index 0000000000..c27331f15b --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/helpers/EntityMap.ts @@ -0,0 +1,222 @@ +import { Readable } from 'stream'; +import sqlite3 from 'sqlite3'; + +class EntityMap { + initialized: boolean; + cols: string[]; + db: any; + + constructor(...args) { + this.initialized = false; + this.cols = args.map((arg) => processKey(arg)); + this.db = new Promise((resolve, reject) => { + const db = new sqlite3.Database(':memory:'); + + const colStatement = + this.cols.length > 0 + ? this.cols.join(' TEXT, ') + ' TEXT' + : 'mappingPlaceholder TEXT'; + db.run(`CREATE TABLE mapping (${colStatement})`, (err) => { + if (err) { + console.log(err); + reject(err); + } + resolve(db); + }); + }); + } + + async init() { + if (!this.initialized) { + this.db = await this.db; + this.initialized = true; + } + } + + destroy() { + if (this.initialized && this.db) { + this.db.close(); + } + } + + async addRow(row) { + if (!this.initialized) { + throw 'Please initialize first!'; + } + + const cols = Object.keys(row).map((key) => processKey(key)); + const colStatement = cols.map((key) => `'${key}'`).join(', '); + const questionMarks = cols.map(() => '?').join(', '); + + const promises = []; + + for (const col of cols.filter((col) => !this.cols.includes(col))) { + promises.push( + new Promise((resolve, reject) => { + this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => { + if (err) { + console.log(err); + reject(err); + } + this.cols.push(col); + resolve(true); + }); + }) + ); + } + + await Promise.all(promises); + + const values = Object.values(row).map((val) => { + if (typeof val === 'object') { + return `JSON::${JSON.stringify(val)}`; + } + return val; + }); + + return new Promise((resolve, reject) => { + this.db.run( + `INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`, + values, + (err) => { + if (err) { + console.log(err); + reject(err); + } + resolve(true); + } + ); + }); + } + + getRow(col, val, res = []): Promise> { + if (!this.initialized) { + throw 'Please initialize first!'; + } + return new Promise((resolve, reject) => { + col = processKey(col); + res = res.map((r) => processKey(r)); + this.db.get( + `SELECT ${ + res.length ? res.join(', ') : '*' + } FROM mapping WHERE ${col} = ?`, + [val], + (err, rs) => { + if (err) { + console.log(err); + reject(err); + } + if (rs) { + rs = processResponseRow(rs); + } + resolve(rs); + } + ); + }); + } + + getCount(): Promise { + if (!this.initialized) { + throw 'Please initialize first!'; + } + return new Promise((resolve, reject) => { + this.db.get(`SELECT COUNT(*) as count FROM mapping`, (err, rs) => { + if (err) { + console.log(err); + reject(err); + } + resolve(rs.count); + }); + }); + } + + getStream(res = []): DBStream { + if (!this.initialized) { + throw 'Please initialize first!'; + } + res = res.map((r) => processKey(r)); + return new DBStream( + this.db, + `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping` + ); + } + + getLimit(limit, offset, res = []): Promise[]> { + if (!this.initialized) { + throw 'Please initialize first!'; + } + return new Promise((resolve, reject) => { + res = res.map((r) => processKey(r)); + this.db.all( + `SELECT ${ + res.length ? res.join(', ') : '*' + } FROM mapping LIMIT ${limit} OFFSET ${offset}`, + (err, rs) => { + if (err) { + console.log(err); + reject(err); + } + for (let row of rs) { + row = processResponseRow(row); + } + resolve(rs); + } + ); + }); + } +} + +class DBStream extends Readable { + db: any; + stmt: any; + sql: any; + + constructor(db, sql) { + super({ objectMode: true }); + this.db = db; + this.sql = sql; + this.stmt = this.db.prepare(this.sql); + this.on('end', () => this.stmt.finalize()); + } + + _read() { + const stream = this; + this.stmt.get(function (err, result) { + if (err) { + stream.emit('error', err); + } else { + if (result) { + result = processResponseRow(result); + } + stream.push(result || null); + } + }); + } +} + +function processResponseRow(res: any) { + for (const key of Object.keys(res)) { + if (res[key] && res[key].startsWith('JSON::')) { + try { + res[key] = JSON.parse(res[key].replace('JSON::', '')); + } catch (e) { + console.log(e); + } + } + if (revertKey(key) !== key) { + res[revertKey(key)] = res[key]; + delete res[key]; + } + } + return res; +} + +function processKey(key) { + return key.replace(/'/g, "''").replace(/[A-Z]/g, (match) => `_${match}`); +} + +function revertKey(key) { + return key.replace(/''/g, "'").replace(/_[A-Z]/g, (match) => match[1]); +} + +export default EntityMap; diff --git a/packages/nocodb-nest/src/modules/sync/helpers/NocoSyncDestAdapter.ts b/packages/nocodb-nest/src/modules/sync/helpers/NocoSyncDestAdapter.ts new file mode 100644 index 0000000000..8af4a493c5 --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/helpers/NocoSyncDestAdapter.ts @@ -0,0 +1,6 @@ +export abstract class NocoSyncSourceAdapter { + public abstract init(): Promise; + public abstract destProjectWrite(): Promise; + public abstract destSchemaWrite(): Promise; + public abstract destDataWrite(): Promise; +} diff --git a/packages/nocodb-nest/src/modules/sync/helpers/NocoSyncSourceAdapter.ts b/packages/nocodb-nest/src/modules/sync/helpers/NocoSyncSourceAdapter.ts new file mode 100644 index 0000000000..a419d1c24d --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/helpers/NocoSyncSourceAdapter.ts @@ -0,0 +1,7 @@ +export abstract class NocoSyncSourceAdapter { + public abstract init(): Promise; + public abstract srcSchemaGet(): Promise; + public abstract srcDataLoad(): Promise; + public abstract srcDataListen(): Promise; + public abstract srcDataPoll(): Promise; +} diff --git a/packages/nocodb-nest/src/modules/sync/helpers/fetchAT.ts b/packages/nocodb-nest/src/modules/sync/helpers/fetchAT.ts new file mode 100644 index 0000000000..702fbd8113 --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/helpers/fetchAT.ts @@ -0,0 +1,238 @@ +import axios from 'axios'; + +const info: any = { + initialized: false, +}; + +async function initialize(shareId) { + info.cookie = ''; + const url = `https://airtable.com/${shareId}`; + + try { + const hreq = await axios + .get(url, { + headers: { + accept: + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'document', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-user': '?1', + 'upgrade-insecure-requests': '1', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + }, + referrerPolicy: 'strict-origin-when-cross-origin', + body: null, + method: 'GET', + }) + .then((response) => { + for (const ck of response.headers['set-cookie']) { + info.cookie += ck.split(';')[0] + '; '; + } + return response.data; + }) + .catch(() => { + throw { + message: + 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + }); + + info.headers = JSON.parse( + hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim() + ); + info.link = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim()); + info.baseInfo = decodeURIComponent(info.link) + .match(/{(.*)}/g)[0] + .split('&') + .reduce((result, el) => { + try { + return Object.assign( + result, + JSON.parse(el.includes('=') ? el.split('=')[1] : el) + ); + } catch (e) { + if (el.includes('=')) { + return Object.assign(result, { + [el.split('=')[0]]: el.split('=')[1], + }); + } + } + }, {}); + info.baseId = info.baseInfo.applicationId; + info.initialized = true; + } catch (e) { + console.log(e); + info.initialized = false; + if (e.message) { + throw e; + } else { + throw { + message: + 'Error processing Shared Base :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + } + } +} + +async function read() { + if (info.initialized) { + const resreq = await axios('https://airtable.com' + info.link, { + headers: { + accept: '*/*', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + 'x-time-zone': 'Europe/Berlin', + cookie: info.cookie, + ...info.headers, + }, + referrerPolicy: 'no-referrer', + body: null, + method: 'GET', + }) + .then((response) => { + return response.data; + }) + .catch(() => { + throw { + message: + 'Error Reading :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + }); + + return { + schema: resreq.data, + baseId: info.baseId, + baseInfo: info.baseInfo, + }; + } else { + throw { + message: 'Error Initializing :: please try again !!', + }; + } +} + +async function readView(viewId) { + if (info.initialized) { + const resreq = await axios( + `https://airtable.com/v0.3/view/${viewId}/readData?` + + `stringifiedObjectParams=${encodeURIComponent('{}')}&requestId=${ + info.baseInfo.requestId + }&accessPolicy=${encodeURIComponent( + JSON.stringify({ + allowedActions: info.baseInfo.allowedActions, + shareId: info.baseInfo.shareId, + applicationId: info.baseInfo.applicationId, + generationNumber: info.baseInfo.generationNumber, + expires: info.baseInfo.expires, + signature: info.baseInfo.signature, + }) + )}`, + { + headers: { + accept: '*/*', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + 'x-time-zone': 'Europe/Berlin', + cookie: info.cookie, + ...info.headers, + }, + referrerPolicy: 'no-referrer', + body: null, + method: 'GET', + } + ) + .then((response) => { + return response.data; + }) + .catch(() => { + throw { + message: + 'Error Reading View :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + }); + return { view: resreq.data }; + } else { + throw { + message: 'Error Initializing :: please try again !!', + }; + } +} + +async function readTemplate(templateId) { + if (!info.initialized) { + await initialize('shrO8aYf3ybwSdDKn'); + } + const resreq = await axios( + `https://www.airtable.com/v0.3/exploreApplications/${templateId}`, + { + headers: { + accept: '*/*', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + 'x-time-zone': 'Europe/Berlin', + cookie: info.cookie, + ...info.headers, + }, + referrer: 'https://www.airtable.com/', + referrerPolicy: 'same-origin', + body: null, + method: 'GET', + mode: 'cors', + credentials: 'include', + } + ) + .then((response) => { + return response.data; + }) + .catch(() => { + throw { + message: + 'Error Fetching :: Ensure www.airtable.com/templates/featured/ is accessible.', + }; + }); + return { template: resreq }; +} + +function unicodeToChar(text) { + return text.replace(/\\u[\dA-F]{4}/gi, function (match) { + return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16)); + }); +} + +export default { + initialize, + read, + readView, + readTemplate, +}; diff --git a/packages/nocodb-nest/src/modules/sync/helpers/job.ts b/packages/nocodb-nest/src/modules/sync/helpers/job.ts new file mode 100644 index 0000000000..2bfb64ca3f --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/helpers/job.ts @@ -0,0 +1,2444 @@ +import { promisify } from 'util'; +import { UITypes } from 'nocodb-sdk'; +import Airtable from 'airtable'; +import jsonfile from 'jsonfile'; +import hash from 'object-hash'; +import { T } from 'nc-help'; +import dayjs from 'dayjs'; +import utc from 'dayjs/plugin/utc'; +import tinycolor from 'tinycolor2'; +import { + attachmentService, + columnService, + filterService, + formViewColumnService, + formViewService, + galleryViewService, + gridViewService, + projectService, + projectUserService, + sortService, + tableService, + viewColumnService, + viewService, +} from '../..'; +import FetchAT from './fetchAT'; +import { importData, importLTARData } from './readAndProcessData'; +import EntityMap from './EntityMap'; +import type { UserType } from 'nocodb-sdk'; + +const writeJsonFileAsync = promisify(jsonfile.writeFile); + +dayjs.extend(utc); + +const selectColors = { + // normal + blue: '#cfdfff', + cyan: '#d0f0fd', + teal: '#c2f5e9', + green: '#d1f7c4', + orange: '#fee2d5', + yellow: '#ffeab6', + red: '#ffdce5', + pink: '#ffdaf6', + purple: '#ede2fe', + gray: '#eee', + // medium + blueMedium: '#9cc7ff', + cyanMedium: '#77d1f3', + tealMedium: '#72ddc3', + greenMedium: '#93e088', + orangeMedium: '#ffa981', + yellowMedium: '#ffd66e', + redMedium: '#ff9eb7', + pinkMedium: '#f99de2', + purpleMedium: '#cdb0ff', + grayMedium: '#ccc', + // dark + blueDark: '#2d7ff9', + cyanDark: '#18bfff', + tealDark: '#20d9d2', + greenDark: '#20c933', + orangeDark: '#ff6f2c', + yellowDark: '#fcb400', + redDark: '#f82b60', + pinkDark: '#ff08c2', + purpleDark: '#8b46ff', + grayDark: '#666', + // darker + blueDarker: '#2750ae', + cyanDarker: '#0b76b7', + tealDarker: '#06a09b', + greenDarker: '#338a17', + orangeDarker: '#d74d26', + yellowDarker: '#b87503', + redDarker: '#ba1e45', + pinkDarker: '#b2158b', + purpleDarker: '#6b1cb0', + grayDarker: '#444', +}; + +export default async ( + syncDB: AirtableSyncConfig, + progress: (data: { msg?: string; level?: any }) => void +) => { + const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent'); + await sMapEM.init(); + const userRole = syncDB.user.roles + .split(',') + .reduce((rolesObj, role) => ({ [role]: true, ...rolesObj }), {}); + + const sMap = { + // static mapping records between aTblId && ncId + async addToMappingTbl(aTblId, ncId, ncName, ncParent?) { + await sMapEM.addRow({ aTblId, ncId, ncName, ncParent }); + }, + + // get NcID from airtable ID + async getNcIdFromAtId(aId) { + return (await sMapEM.getRow('aTblId', aId, ['ncId']))?.ncId; + }, + + // get nc Parent from airtable ID + async getNcParentFromAtId(aId) { + return (await sMapEM.getRow('aTblId', aId, ['ncParent']))?.ncParent; + }, + + // get nc-title from airtable ID + async getNcNameFromAtId(aId) { + return (await sMapEM.getRow('aTblId', aId, ['ncName']))?.ncName; + }, + }; + + function logBasic(log) { + progress({ level: 0, msg: log }); + } + + function logDetailed(log) { + if (debugMode) progress({ level: 1, msg: log }); + } + + const perfStats = []; + function recordPerfStart() { + if (!debugMode) return 0; + return Date.now(); + } + function recordPerfStats(start, event) { + if (!debugMode) return; + const duration = Date.now() - start; + perfStats.push({ d: duration, e: event }); + } + + let base, baseId; + const start = Date.now(); + const enableErrorLogs = false; + const generate_migrationStats = true; + const debugMode = false; + let g_aTblSchema = []; + let ncCreatedProjectSchema: any = {}; + const ncLinkMappingTable: any[] = []; + const nestedLookupTbl: any[] = []; + const nestedRollupTbl: any[] = []; + const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' }; + const storeLinks = false; + const ncLinkDataStore: any = {}; + const insertedAssocRef: any = {}; + + const atNcAliasRef: { + [ncTableId: string]: { + [ncTitle: string]: string; + }; + } = {}; + + const uniqueTableNameGen = getUniqueNameGenerator('sheet'); + + // run time counter (statistics) + const rtc = { + sort: 0, + filter: 0, + view: { + total: 0, + grid: 0, + gallery: 0, + form: 0, + }, + fetchAt: { + count: 0, + time: 0, + }, + migrationSkipLog: { + count: 0, + log: [], + }, + data: { + records: 0, + nestedLinks: 0, + }, + }; + + function updateMigrationSkipLog(tbl, col, type, reason?) { + rtc.migrationSkipLog.count++; + rtc.migrationSkipLog.log.push( + `tn[${tbl}] cn[${col}] type[${type}] :: ${reason}` + ); + } + + // mapping table + // + + async function getAirtableSchema(sDB) { + const start = Date.now(); + + if (!sDB.shareId) + throw { + message: + 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + + if (sDB.shareId.startsWith('exp')) { + const template = await FetchAT.readTemplate(sDB.shareId); + await FetchAT.initialize(template.template.exploreApplication.shareId); + } else { + await FetchAT.initialize(sDB.shareId); + } + const ft = await FetchAT.read(); + const duration = Date.now() - start; + rtc.fetchAt.count++; + rtc.fetchAt.time += duration; + + if (!ft.baseId) { + throw { + message: + 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + } + + const file = ft.schema; + baseId = ft.baseId; + base = new Airtable({ apiKey: sDB.apiKey }).base(baseId); + // store copy of airtable schema globally + g_aTblSchema = file.tableSchemas; + + if (debugMode) + await writeJsonFileAsync('aTblSchema.json', ft, { spaces: 2 }); + + return file; + } + + async function getViewData(viewId) { + const start = Date.now(); + const ft = await FetchAT.readView(viewId); + const duration = Date.now() - start; + rtc.fetchAt.count++; + rtc.fetchAt.time += duration; + + if (debugMode) + await writeJsonFileAsync(`${viewId}.json`, ft, { spaces: 2 }); + return ft.view; + } + + function getRootDbType() { + return ncCreatedProjectSchema?.bases.find((el) => el.id === syncDB.baseId) + ?.type; + } + + // base mapping table + const aTblNcTypeMap = { + foreignKey: UITypes.LinkToAnotherRecord, + text: UITypes.SingleLineText, + multilineText: UITypes.LongText, + richText: UITypes.LongText, + multipleAttachment: UITypes.Attachment, + checkbox: UITypes.Checkbox, + multiSelect: UITypes.MultiSelect, + select: UITypes.SingleSelect, + collaborator: UITypes.Collaborator, + multiCollaborator: UITypes.Collaborator, + date: UITypes.Date, + phone: UITypes.PhoneNumber, + number: UITypes.Decimal, + rating: UITypes.Rating, + formula: UITypes.Formula, + rollup: UITypes.Rollup, + count: UITypes.Count, + lookup: UITypes.Lookup, + autoNumber: UITypes.AutoNumber, + barcode: UITypes.SingleLineText, + button: UITypes.Button, + }; + + //----------------------------------------------------------------------------- + // aTbl helper routines + // + + function nc_sanitizeName(name) { + // replace all special characters by _ + return name.replace(/\W+/g, '_').trim(); + } + + function nc_getSanitizedColumnName(table, name) { + let col_name = nc_sanitizeName(name); + + // truncate to 60 chars if character if exceeds above 60 + col_name = col_name?.slice(0, 60); + + // for knex, replace . with _ + const col_alias = name.trim().replace(/\./g, '_'); + + // check if already a column exists with same name? + const duplicateTitle = table.columns.find( + (x) => x.title?.toLowerCase() === col_alias?.toLowerCase() + ); + const duplicateColumn = table.columns.find( + (x) => x.column_name?.toLowerCase() === col_name?.toLowerCase() + ); + if (duplicateTitle) { + if (enableErrorLogs) console.log(`## Duplicate title ${col_alias}`); + } + + return { + // kludge: error observed in Nc with space around column-name + title: col_alias + (duplicateTitle ? '_2' : ''), + column_name: col_name + (duplicateColumn ? '_2' : ''), + }; + } + + const ncSchema = { + tables: [], + tablesById: {}, + }; + + // aTbl: retrieve column name from column ID + // + function aTbl_getColumnName(colId): any { + for (let i = 0; i < g_aTblSchema.length; i++) { + const sheetObj = g_aTblSchema[i]; + const column = sheetObj.columns.find((col) => col.id === colId); + if (column !== undefined) + return { + tn: sheetObj.name, + cn: column.name, + }; + } + } + + // nc dump schema + // + // @ts-ignore + async function nc_DumpTableSchema() { + console.log('['); + // const ncTblList = await api.base.tableList( + // ncCreatedProjectSchema.id, + // syncDB.baseId + // ); + + const ncTblList = { list: [] }; + ncTblList['list'] = await tableService.getAccessibleTables({ + projectId: ncCreatedProjectSchema.id, + baseId: syncDB.baseId, + roles: userRole, + }); + + for (let i = 0; i < ncTblList.list.length; i++) { + // const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + const ncTbl = await tableService.getTableWithAccessibleViews({ + tableId: ncTblList.list[i].id, + user: syncDB.user, + }); + console.log(JSON.stringify(ncTbl, null, 2)); + console.log(','); + } + console.log(']'); + } + + // retrieve nc column schema from using aTbl field ID as reference + // + async function nc_getColumnSchema(aTblFieldId) { + const ncTblId = await sMap.getNcParentFromAtId(aTblFieldId); + const ncColId = await sMap.getNcIdFromAtId(aTblFieldId); + + // not migrated column, skip + if (ncColId === undefined || ncTblId === undefined) return 0; + + return ncSchema.tablesById[ncTblId].columns.find((x) => x.id === ncColId); + } + + // retrieve nc table schema using table name + // optimize: create a look-up table & re-use information + // + async function nc_getTableSchema(tableName) { + return ncSchema.tables.find((x) => x.title === tableName); + } + + // delete project if already exists + async function init({ + projectName, + }: { + projectName?: string; + projectId?: string; + }) { + // delete 'sample' project if already exists + const x = { list: [] }; + x['list'] = await projectService.projectList({ + user: { id: syncDB.user.id, roles: syncDB.user.roles }, + }); + + const sampleProj = x.list.find((a) => a.title === projectName); + if (sampleProj) { + await projectService.projectSoftDelete({ + projectId: sampleProj.id, + }); + } + logDetailed('Init'); + } + + // map UIDT + // + function getNocoType(col) { + // start with default map + let ncType = aTblNcTypeMap[col.type]; + + // types email & url are marked as text + // types currency & percent, duration are marked as number + // types createTime & modifiedTime are marked as formula + + switch (col.type) { + case 'text': + if (col.typeOptions?.validatorName === 'email') ncType = UITypes.Email; + else if (col.typeOptions?.validatorName === 'url') ncType = UITypes.URL; + break; + + case 'number': + // kludge: currency validation error with decimal places + if (col.typeOptions?.format === 'percentV2') ncType = UITypes.Percent; + else if (col.typeOptions?.format === 'duration') + ncType = UITypes.Duration; + else if (col.typeOptions?.format === 'currency') + ncType = UITypes.Currency; + else if (col.typeOptions?.precision > 0) ncType = UITypes.Decimal; + break; + + case 'formula': + if (col.typeOptions?.formulaTextParsed === 'CREATED_TIME()') + ncType = UITypes.DateTime; + else if (col.typeOptions?.formulaTextParsed === 'LAST_MODIFIED_TIME()') + ncType = UITypes.DateTime; + break; + + case 'computation': + if (col.typeOptions?.resultType === 'collaborator') + ncType = UITypes.Collaborator; + break; + + case 'date': + if (col.typeOptions?.isDateTime) ncType = UITypes.DateTime; + break; + } + + return ncType; + } + + // retrieve additional options associated with selected data types + // + async function getNocoTypeOptions(col: any): Promise { + switch (col.type) { + case 'select': + case 'multiSelect': { + // prepare options list in CSV format + // note: NC doesn't allow comma's in options + // + const options = []; + let order = 1; + for (const [, value] of Object.entries(col.typeOptions.choices)) { + // replace commas with dot for multiselect + if (col.type === 'multiSelect') { + (value as any).name = (value as any).name.replace(/,/g, '.'); + } + // we don't allow empty records, placeholder instead + if ((value as any).name === '') { + (value as any).name = 'nc_empty'; + } + // enumerate duplicates (we don't allow them) + // TODO fix record mapping (this causes every record to map first option, + // we can't handle them using data api as they don't provide option id + // within data we might instead get the correct mapping from schema file ) + let dupNo = 1; + const defaultName = (value as any).name; + while ( + options.find( + (el) => + el.title.toLowerCase() === (value as any).name.toLowerCase() + ) + ) { + (value as any).name = `${defaultName}_${dupNo++}`; + } + options.push({ + order: order++, + title: (value as any).name, + color: selectColors[(value as any).color] + ? selectColors[(value as any).color] + : tinycolor.random().toHexString(), + }); + + await sMap.addToMappingTbl( + (value as any).id, + undefined, + (value as any).name + ); + } + return { type: col.type, data: options }; + } + default: + return { type: undefined }; + } + } + + // convert to Nc schema (basic, excluding relations) + // + async function tablesPrepare(tblSchema: any[]) { + const tables: any[] = []; + + for (let i = 0; i < tblSchema.length; ++i) { + const table: any = {}; + + if (syncDB.options.syncViews) { + rtc.view.total += tblSchema[i].views.reduce( + (acc, cur) => + ['grid', 'form', 'gallery'].includes(cur.type) ? ++acc : acc, + 0 + ); + } else { + rtc.view.total = tblSchema.length; + } + + // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id; + table.title = tblSchema[i].name; + let sanitizedName = nc_sanitizeName(tblSchema[i].name); + + // truncate to 50 chars if character if exceeds above 50 + // upto 64 should be fine but we are keeping it to 50 since + // meta project adds prefix as well + sanitizedName = sanitizedName?.slice(0, 50); + + // check for duplicate and populate a unique name if already exist + table.table_name = uniqueTableNameGen(sanitizedName); + + const uniqueColNameGen = getUniqueNameGenerator('field'); + table.columns = []; + const sysColumns = [ + { + title: ncSysFields.id, + column_name: ncSysFields.id, + uidt: UITypes.ID, + meta: { + ag: 'nc', + }, + }, + { + title: ncSysFields.hash, + column_name: ncSysFields.hash, + uidt: UITypes.SingleLineText, + system: true, + }, + ]; + + for (let j = 0; j < tblSchema[i].columns.length; j++) { + const col = tblSchema[i].columns[j]; + + // skip link, lookup, rollup fields in this iteration + if (['foreignKey', 'lookup', 'rollup'].includes(col.type)) { + continue; + } + + // base column schema + const ncName: any = nc_getSanitizedColumnName(table, col.name); + const ncCol: any = { + // Enable to use aTbl identifiers as is: id: col.id, + title: ncName.title, + column_name: uniqueColNameGen(ncName.column_name), + uidt: getNocoType(col), + }; + + // not supported datatype: pure formula field + // allow formula based computed fields (created time/ modified time to go through) + if (ncCol.uidt === UITypes.Formula) { + updateMigrationSkipLog( + tblSchema[i].name, + ncName.title, + col.type, + 'column type not supported' + ); + continue; + } + + // change from default 'tinytext' as airtable allows more than 255 characters + // for single line text column type + if (col.type === 'text') ncCol.dt = 'text'; + + // #fix-2363-decimal-out-of-range + if (['sqlite3', 'mysql2'].includes(getRootDbType())) { + if (ncCol.uidt === UITypes.Decimal) { + ncCol.dt = 'double'; + ncCol.dtxp = 22; + ncCol.dtxs = '2'; + } + } + + // additional column parameters when applicable + const colOptions = await getNocoTypeOptions(col); + + switch (colOptions.type) { + case 'select': + case 'multiSelect': + ncCol.colOptions = { + options: [...colOptions.data], + }; + + if (['mysql', 'mysql2'].includes(getRootDbType())) { + // if options are empty, configure '' as an option + ncCol.dtxp = + colOptions.data + .map((el) => `'${el.title.replace(/'/gi, "''")}'`) + .join(',') || "''"; + } + + break; + case undefined: + break; + } + table.columns.push(ncCol); + } + table.columns.push(sysColumns[0]); + table.columns.push(sysColumns[1]); + + tables.push(table); + } + return tables; + } + + async function nocoCreateBaseSchema(aTblSchema) { + // base schema preparation: exclude + const tables: any[] = await tablesPrepare(aTblSchema); + + // for each table schema, create nc table + for (let idx = 0; idx < tables.length; idx++) { + logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`); + + logDetailed(`NC API: base.tableCreate ${tables[idx].title}`); + + let _perfStart = recordPerfStart(); + const table = await tableService.tableCreate({ + baseId: syncDB.baseId, + projectId: ncCreatedProjectSchema.id, + table: tables[idx], + user: syncDB.user, + }); + recordPerfStats(_perfStart, 'dbTable.create'); + + updateNcTblSchema(table); + + // update mapping table + await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title); + for (let colIdx = 0; colIdx < table.columns.length; colIdx++) { + const aId = aTblSchema[idx].columns.find( + (x) => + x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title + )?.id; + if (aId) + await sMap.addToMappingTbl( + aId, + table.columns[colIdx].id, + table.columns[colIdx].title, + table.id + ); + } + + // update default view name- to match it to airtable view name + logDetailed(`NC API: dbView.list ${table.id}`); + _perfStart = recordPerfStart(); + const view = { list: [] }; + view['list'] = await viewService.viewList({ + tableId: table.id, + user: { roles: userRole }, + }); + recordPerfStats(_perfStart, 'dbView.list'); + + const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid'); + logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); + _perfStart = recordPerfStart(); + await viewService.viewUpdate({ + viewId: view.list[0].id, + view: { title: aTbl_grid.name }, + }); + recordPerfStats(_perfStart, 'dbView.update'); + + await updateNcTblSchemaById(table.id); + + await sMap.addToMappingTbl( + aTbl_grid.id, + table.views[0].id, + aTbl_grid.name, + table.id + ); + } + + return tables; + } + + async function nocoCreateLinkToAnotherRecord(aTblSchema) { + // Link to another RECORD + for (let idx = 0; idx < aTblSchema.length; idx++) { + const aTblLinkColumns = aTblSchema[idx].columns.filter( + (x) => x.type === 'foreignKey' + ); + + // Link columns exist + // + if (aTblLinkColumns.length) { + for (let i = 0; i < aTblLinkColumns.length; i++) { + logDetailed( + `[${idx + 1}/${aTblSchema.length}] Configuring Links :: [${i + 1}/${ + aTblLinkColumns.length + }] ${aTblSchema[idx].name}` + ); + + // for self links, there is no symmetric column + { + const src = aTbl_getColumnName(aTblLinkColumns[i].id); + const dst = aTbl_getColumnName( + aTblLinkColumns[i].typeOptions?.symmetricColumnId + ); + logDetailed( + `LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst?.tn}:${dst?.cn}` + ); + } + + // check if link already established? + if (!nc_isLinkExists(aTblLinkColumns[i].id)) { + // parent table ID + const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + + // find child table name from symmetric column ID specified + // self link, symmetricColumnId field will be undefined + const childTable = aTbl_getColumnName( + aTblLinkColumns[i].typeOptions?.symmetricColumnId + ); + + // retrieve child table ID (nc) from table name + let childTableId = srcTableId; + if (childTable) { + childTableId = (await nc_getTableSchema(childTable.tn)).id; + } + + // check if already a column exists with this name? + let _perfStart = recordPerfStart(); + const srcTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: srcTableId, + user: syncDB.user, + }); + recordPerfStats(_perfStart, 'dbTable.read'); + + // create link + const ncName = nc_getSanitizedColumnName( + srcTbl, + aTblLinkColumns[i].name + ); + + // LTAR alias ref to AT + atNcAliasRef[srcTbl.id] = atNcAliasRef[srcTbl.id] || {}; + atNcAliasRef[srcTbl.id][ncName.title] = aTblLinkColumns[i].name; + + logDetailed( + `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` + ); + _perfStart = recordPerfStart(); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.LinkToAnotherRecord, + title: ncName.title, + column_name: ncName.column_name, + parentId: srcTableId, + childId: childTableId, + type: 'mm', + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === ncName.title + )?.id; + await sMap.addToMappingTbl( + aTblLinkColumns[i].id, + ncId, + ncName.title, + ncTbl.id + ); + + // store link information in separate table + // this information will be helpful in identifying relation pair + const link = { + nc: { + title: ncName.title, + parentId: srcTableId, + childId: childTableId, + type: 'mm', + }, + aTbl: { + tblId: aTblSchema[idx].id, + ...aTblLinkColumns[i], + }, + }; + + ncLinkMappingTable.push(link); + } else { + // if link already exists, we need to change name of linked column + // to what is represented in airtable + + // 1. extract associated link information from link table + // 2. retrieve parent table information (source) + // 3. using foreign parent & child column ID, find associated mapping in child table + // 4. update column name + const x = ncLinkMappingTable.findIndex( + (x) => + x.aTbl.tblId === + aTblLinkColumns[i].typeOptions.foreignTableId && + x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId + ); + + let _perfStart = recordPerfStart(); + const childTblSchema: any = + await tableService.getTableWithAccessibleViews({ + tableId: ncLinkMappingTable[x].nc.childId, + user: syncDB.user, + }); + recordPerfStats(_perfStart, 'dbTable.read'); + + _perfStart = recordPerfStart(); + const parentTblSchema: any = + await tableService.getTableWithAccessibleViews({ + tableId: ncLinkMappingTable[x].nc.parentId, + user: syncDB.user, + }); + recordPerfStats(_perfStart, 'dbTable.read'); + + let parentLinkColumn = parentTblSchema.columns.find( + (col) => col.title === ncLinkMappingTable[x].nc.title + ); + + if (parentLinkColumn === undefined) { + updateMigrationSkipLog( + parentTblSchema?.title, + ncLinkMappingTable[x].nc.title, + UITypes.LinkToAnotherRecord, + 'Link error' + ); + continue; + } + + // hack // fix me + if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') { + parentLinkColumn = parentTblSchema.columns.find( + (col) => col.title === ncLinkMappingTable[x].nc.title + '_2' + ); + } + + let childLinkColumn: any = {}; + + if (parentLinkColumn.colOptions.type == 'hm') { + // for hm: + // mapping between child & parent column id is direct + // + childLinkColumn = childTblSchema.columns.find( + (col) => + col.uidt === UITypes.LinkToAnotherRecord && + col.colOptions.fk_child_column_id === + parentLinkColumn.colOptions.fk_child_column_id && + col.colOptions.fk_parent_column_id === + parentLinkColumn.colOptions.fk_parent_column_id + ); + } else { + // for mm: + // mapping between child & parent column id is inverted + // + childLinkColumn = childTblSchema.columns.find( + (col) => + col.uidt === UITypes.LinkToAnotherRecord && + col.colOptions.fk_child_column_id === + parentLinkColumn.colOptions.fk_parent_column_id && + col.colOptions.fk_parent_column_id === + parentLinkColumn.colOptions.fk_child_column_id && + col.colOptions.fk_mm_model_id === + parentLinkColumn.colOptions.fk_mm_model_id + ); + } + + // check if already a column exists with this name? + const duplicate = childTblSchema.columns.find( + (x) => x.title === aTblLinkColumns[i].name + ); + const suffix = duplicate ? '_2' : ''; + if (duplicate) + if (enableErrorLogs) + console.log(`## Duplicate ${aTblLinkColumns[i].name}`); + + // rename + // note that: current rename API requires us to send all parameters, + // not just title being renamed + const ncName = nc_getSanitizedColumnName( + childTblSchema, + aTblLinkColumns[i].name + ); + + logDetailed( + `NC API: dbTableColumn.update rename symmetric column ${ncName.title}` + ); + _perfStart = recordPerfStart(); + const ncTbl: any = await columnService.columnUpdate({ + columnId: childLinkColumn.id, + column: { + ...childLinkColumn, + title: ncName.title, + column_name: ncName.column_name, + }, + }); + recordPerfStats(_perfStart, 'dbTableColumn.update'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === aTblLinkColumns[i].name + suffix + )?.id; + await sMap.addToMappingTbl( + aTblLinkColumns[i].id, + ncId, + aTblLinkColumns[i].name + suffix, + ncTbl.id + ); + } + } + } + } + } + + async function nocoCreateLookups(aTblSchema) { + // LookUps + for (let idx = 0; idx < aTblSchema.length; idx++) { + const aTblColumns = aTblSchema[idx].columns.filter( + (x) => x.type === 'lookup' + ); + + // parent table ID + const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + if (aTblColumns.length) { + // Lookup + for (let i = 0; i < aTblColumns.length; i++) { + logDetailed( + `[${idx + 1}/${aTblSchema.length}] Configuring Lookup :: [${ + i + 1 + }/${aTblColumns.length}] ${aTblSchema[idx].name}` + ); + + // something is not right, skip + if ( + aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + if (enableErrorLogs) + console.log(`## Invalid column IDs mapped; skip`); + + updateMigrationSkipLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + 'invalid column ID in dependency list' + ); + continue; + } + + const ncRelationColumnId = await sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.relationColumnId + ); + const ncLookupColumnId = await sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + + if ( + ncLookupColumnId === undefined || + ncRelationColumnId === undefined + ) { + aTblColumns[i]['srcTableId'] = srcTableId; + nestedLookupTbl.push(aTblColumns[i]); + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + aTblColumns[i].name + ); + + logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); + const _perfStart = recordPerfStart(); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === aTblColumns[i].name + )?.id; + await sMap.addToMappingTbl( + aTblColumns[i].id, + ncId, + aTblColumns[i].name, + ncTbl.id + ); + } + } + } + + let level = 2; + let nestedCnt = 0; + while (nestedLookupTbl.length) { + // if nothing has changed from previous iteration, skip rest + if (nestedCnt === nestedLookupTbl.length) { + for (let i = 0; i < nestedLookupTbl.length; i++) { + const fTblField = + nestedLookupTbl[i].typeOptions.foreignTableRollupColumnId; + const name = aTbl_getColumnName(fTblField); + updateMigrationSkipLog( + ncSchema.tablesById[nestedLookupTbl[i].srcTableId]?.title, + nestedLookupTbl[i].name, + nestedLookupTbl[i].type, + `foreign table field not found [${name.tn}/${name.cn}]` + ); + } + if (enableErrorLogs) + console.log( + `## Failed to configure ${nestedLookupTbl.length} lookups` + ); + break; + } + + // Nested lookup + nestedCnt = nestedLookupTbl.length; + for (let i = 0; i < nestedLookupTbl.length; i++) { + const srcTableId = nestedLookupTbl[0].srcTableId; + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + const ncRelationColumnId = await sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.relationColumnId + ); + const ncLookupColumnId = await sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId + ); + + if ( + ncLookupColumnId === undefined || + ncRelationColumnId === undefined + ) { + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + nestedLookupTbl[0].name + ); + + logDetailed( + `Configuring Nested Lookup: Level-${level} [${i + 1}/${nestedCnt} ${ + ncName.title + }]` + ); + + logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); + const _perfStart = recordPerfStart(); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === nestedLookupTbl[0].name + )?.id; + await sMap.addToMappingTbl( + nestedLookupTbl[0].id, + ncId, + nestedLookupTbl[0].name, + ncTbl.id + ); + + // remove entry + nestedLookupTbl.splice(0, 1); + } + level++; + } + } + + function getRollupNcFunction(aTblFunction) { + const fn = aTblFunction.split('(')[0]; + const aTbl_ncRollUp = { + AND: '', + ARRAYCOMPACT: '', + ARRAYJOIN: '', + ARRAYUNIQUE: '', + AVERAGE: 'average', + CONCATENATE: '', + COUNT: 'count', + COUNTA: '', + COUNTALL: '', + MAX: 'max', + MIN: 'min', + OR: '', + SUM: 'sum', + XOR: '', + }; + return aTbl_ncRollUp[fn]; + } + + //@ts-ignore + async function nocoCreateRollup(aTblSchema) { + // Rollup + for (let idx = 0; idx < aTblSchema.length; idx++) { + const aTblColumns = aTblSchema[idx].columns.filter( + (x) => x.type === 'rollup' + ); + + // parent table ID + const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + if (aTblColumns.length) { + // rollup exist + for (let i = 0; i < aTblColumns.length; i++) { + logDetailed( + `[${idx + 1}/${aTblSchema.length}] Configuring Rollup :: [${ + i + 1 + }/${aTblColumns.length}] ${aTblSchema[idx].name}` + ); + + // fetch associated rollup function + // skip column creation if supported rollup function does not exist + const ncRollupFn = getRollupNcFunction( + aTblColumns[i].typeOptions.formulaTextParsed + ); + + if (ncRollupFn === '' || ncRollupFn === undefined) { + updateMigrationSkipLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + `rollup function ${aTblColumns[i].typeOptions.formulaTextParsed} not supported` + ); + continue; + } + + // something is not right, skip + if ( + aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + if (enableErrorLogs) + console.log(`## Invalid column IDs mapped; skip`); + + updateMigrationSkipLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + 'invalid column ID in dependency list' + ); + continue; + } + + const ncRelationColumnId = await sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.relationColumnId + ); + const ncRollupColumnId = await sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + + if (ncRollupColumnId === undefined) { + aTblColumns[i]['srcTableId'] = srcTableId; + nestedRollupTbl.push(aTblColumns[i]); + continue; + } + + // skip, if rollup column was pointing to another virtual column + const ncColSchema = await nc_getColumnSchema( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + if ( + ncColSchema?.uidt === UITypes.Formula || + ncColSchema?.uidt === UITypes.Lookup || + ncColSchema?.uidt === UITypes.Rollup || + ncColSchema?.uidt === UITypes.Checkbox + ) { + updateMigrationSkipLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + 'rollup referring to a column type not supported currently' + ); + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + aTblColumns[i].name + ); + + logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); + const _perfStart = recordPerfStart(); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Rollup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_rollup_column_id: ncRollupColumnId, + rollup_function: ncRollupFn, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === aTblColumns[i].name + )?.id; + await sMap.addToMappingTbl( + aTblColumns[i].id, + ncId, + aTblColumns[i].name, + ncTbl.id + ); + } + } + } + logDetailed(`Nested rollup: ${nestedRollupTbl.length}`); + } + + //@ts-ignore + async function nocoLookupForRollup() { + const nestedCnt = nestedLookupTbl.length; + for (let i = 0; i < nestedLookupTbl.length; i++) { + const srcTableId = nestedLookupTbl[0].srcTableId; + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + const ncRelationColumnId = await sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.relationColumnId + ); + const ncLookupColumnId = await sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId + ); + + if (ncLookupColumnId === undefined || ncRelationColumnId === undefined) { + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + nestedLookupTbl[0].name + ); + + logDetailed( + `Configuring Lookup over Rollup :: [${i + 1}/${nestedCnt}] ${ + ncName.title + }` + ); + + logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); + const _perfStart = recordPerfStart(); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === nestedLookupTbl[0].name + )?.id; + await sMap.addToMappingTbl( + nestedLookupTbl[0].id, + ncId, + nestedLookupTbl[0].name, + ncTbl.id + ); + + // remove entry + nestedLookupTbl.splice(0, 1); + } + } + + async function nocoSetPrimary(aTblSchema) { + for (let idx = 0; idx < aTblSchema.length; idx++) { + logDetailed( + `[${idx + 1}/${aTblSchema.length}] Configuring Display value : ${ + aTblSchema[idx].name + }` + ); + + const pColId = aTblSchema[idx].primaryColumnId; + const ncColId = await sMap.getNcIdFromAtId(pColId); + + // skip primary column configuration if we field not migrated + if (ncColId) { + logDetailed(`NC API: dbTableColumn.primaryColumnSet`); + const _perfStart = recordPerfStart(); + await columnService.columnSetAsPrimary({ columnId: ncColId }); + recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet'); + + // update schema + const ncTblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + await updateNcTblSchemaById(ncTblId); + } + } + } + + // retrieve nc-view column ID from corresponding nc-column ID + async function nc_getViewColumnId(viewId, viewType, ncColumnId) { + // retrieve view Info + let viewDetails; + + const _perfStart = recordPerfStart(); + if (viewType === 'form') { + viewDetails = (await formViewService.formViewGet({ formViewId: viewId })) + .columns; + recordPerfStats(_perfStart, 'dbView.formRead'); + } else if (viewType === 'gallery') { + viewDetails = ( + await galleryViewService.galleryViewGet({ galleryViewId: viewId }) + ).columns; + recordPerfStats(_perfStart, 'dbView.galleryRead'); + } else { + viewDetails = await viewColumnService.columnList({ viewId: viewId }); + recordPerfStats(_perfStart, 'dbView.gridColumnsList'); + } + + return viewDetails.find((x) => x.fk_column_id === ncColumnId)?.id; + } + + ////////// Data processing + + async function nocoBaseDataProcessing_v2(sDB, table, record) { + const recordHash = hash(record); + const rec = { ...record.fields }; + + // kludge - + // trim spaces on either side of column name + // leads to error in NocoDB + Object.keys(rec).forEach((key) => { + const replacedKey = key.trim().replace(/\./g, '_'); + if (key !== replacedKey) { + rec[replacedKey] = rec[key]; + delete rec[key]; + } + }); + + // post-processing on the record + for (const [key, value] of Object.entries(rec as { [key: string]: any })) { + // retrieve datatype + const dt = table.columns.find((x) => x.title === key)?.uidt; + + // always process LTAR, Lookup, and Rollup columns as we delete the key after processing + if ( + !value && + dt !== UITypes.LinkToAnotherRecord && + dt !== UITypes.Lookup && + dt !== UITypes.Rollup + ) { + rec[key] = null; + continue; + } + + switch (dt) { + // https://www.npmjs.com/package/validator + // default value: digits_after_decimal: [2] + // if currency, set decimal place to 2 + // + case UITypes.Currency: + rec[key] = (+value).toFixed(2); + break; + + // we will pick up LTAR once all table data's are in place + case UITypes.LinkToAnotherRecord: + if (storeLinks) { + if (ncLinkDataStore[table.title][record.id] === undefined) + ncLinkDataStore[table.title][record.id] = { + id: record.id, + fields: {}, + }; + ncLinkDataStore[table.title][record.id]['fields'][key] = value; + } + delete rec[key]; + break; + + // these will be automatically populated depending on schema configuration + case UITypes.Lookup: + case UITypes.Rollup: + delete rec[key]; + break; + + case UITypes.Collaborator: + // in case of multi-collaborator, this will be an array + if (Array.isArray(value)) { + let collaborators = ''; + for (let i = 0; i < value.length; i++) { + collaborators += `${value[i]?.name} <${value[i]?.email}>, `; + rec[key] = collaborators; + } + } else rec[key] = `${value?.name} <${value?.email}>`; + break; + + case UITypes.Button: + rec[key] = `${value?.label} <${value?.url}>`; + break; + + case UITypes.DateTime: + case UITypes.CreateTime: + case UITypes.LastModifiedTime: + rec[key] = dayjs(value).format('YYYY-MM-DD HH:mm'); + break; + + case UITypes.Date: + if (/\d{5,}/.test(value)) { + // skip + rec[key] = null; + logBasic(`:: Invalid date ${value}`); + } else { + rec[key] = dayjs(value).format('YYYY-MM-DD'); + } + break; + + case UITypes.SingleSelect: + if (value === '') { + rec[key] = 'nc_empty'; + } + rec[key] = value; + break; + + case UITypes.MultiSelect: + rec[key] = value + ?.map((v) => { + if (v === '') { + return 'nc_empty'; + } + return `${v.replace(/,/g, '.')}`; + }) + .join(','); + break; + + case UITypes.Attachment: + if (!syncDB.options.syncAttachment) rec[key] = null; + else { + let tempArr = []; + + try { + logBasic( + ` :: Retrieving attachment :: ${value + ?.map((a) => a.filename?.split('?')?.[0]) + .join(', ')}` + ); + tempArr = await attachmentService.uploadViaURL({ + path: `noco/${sDB.projectName}/${table.title}/${key}`, + urls: value?.map((attachment) => ({ + fileName: attachment.filename?.split('?')?.[0], + url: attachment.url, + size: attachment.size, + mimetype: attachment.type, + })), + }); + } catch (e) { + console.log(e); + } + + rec[key] = JSON.stringify(tempArr); + } + break; + + case UITypes.SingleLineText: + // Barcode data + if (value?.text) { + rec[key] = value.text; + } + break; + + default: + break; + } + } + + // insert airtable record ID explicitly into each records + rec[ncSysFields.id] = record.id; + rec[ncSysFields.hash] = recordHash; + + return rec; + } + + // @ts-ignore + async function nocoReadDataSelected(projName, table, callback, fields) { + return new Promise((resolve, reject) => { + base(table.title) + .select({ + pageSize: 100, + // maxRecords: 100, + fields: fields, + }) + .eachPage( + async function page(records, fetchNextPage) { + // This function (`page`) will get called for each page of records. + // records.forEach(record => callback(table, record)); + logBasic( + `:: ${table.title} / ${fields} : ${ + recordCnt + 1 + } ~ ${(recordCnt += 100)}` + ); + await Promise.all( + records.map((r) => callback(projName, table, r, fields)) + ); + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + function done(err) { + if (err) { + console.error(err); + reject(err); + } + resolve(null); + } + ); + }); + } + + ////////// + + function nc_isLinkExists(airtableFieldId) { + return !!ncLinkMappingTable.find( + (x) => x.aTbl.typeOptions.symmetricColumnId === airtableFieldId + ); + } + + async function nocoCreateProject(projName) { + // create empty project (XC-DB) + logDetailed(`Create Project: ${projName}`); + const _perfStart = recordPerfStart(); + + ncCreatedProjectSchema = await projectService.projectCreate({ + project: { title: projName }, + user: { id: syncDB.user.id }, + }); + + recordPerfStats(_perfStart, 'project.create'); + } + + async function nocoGetProject(projId) { + // create empty project (XC-DB) + logDetailed(`Getting project meta: ${projId}`); + const _perfStart = recordPerfStart(); + ncCreatedProjectSchema = await projectService.getProjectWithInfo({ + projectId: projId, + }); + recordPerfStats(_perfStart, 'project.read'); + } + + async function nocoConfigureGalleryView(sDB, aTblSchema) { + if (!sDB.options.syncViews) return; + for (let idx = 0; idx < aTblSchema.length; idx++) { + const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + const galleryViews = aTblSchema[idx].views.filter( + (x) => x.type === 'gallery' + ); + + const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; + rtc.view.gallery += galleryViews.length; + + for (let i = 0; i < galleryViews.length; i++) { + logDetailed(` Axios fetch view-data`); + + // create view + await getViewData(galleryViews[i].id); + const viewName = aTblSchema[idx].views.find( + (x) => x.id === galleryViews[i].id + )?.name; + + logBasic( + `:: [${configuredViews + i + 1}/${rtc.view.total}] Gallery : ${ + aTblSchema[idx].name + } / ${viewName}` + ); + + logDetailed(`NC API dbView.galleryCreate :: ${viewName}`); + const _perfStart = recordPerfStart(); + await galleryViewService.galleryViewCreate({ + tableId: tblId, + gallery: { + title: viewName, + }, + }); + recordPerfStats(_perfStart, 'dbView.galleryCreate'); + + await updateNcTblSchemaById(tblId); + } + } + } + + async function nocoConfigureFormView(sDB, aTblSchema) { + if (!sDB.options.syncViews) return; + for (let idx = 0; idx < aTblSchema.length; idx++) { + const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + const formViews = aTblSchema[idx].views.filter((x) => x.type === 'form'); + + const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; + rtc.view.form += formViews.length; + for (let i = 0; i < formViews.length; i++) { + logDetailed(` Axios fetch view-data`); + + // create view + const vData = await getViewData(formViews[i].id); + const viewName = aTblSchema[idx].views.find( + (x) => x.id === formViews[i].id + )?.name; + + logBasic( + `:: [${configuredViews + i + 1}/${rtc.view.total}] Form : ${ + aTblSchema[idx].name + } / ${viewName}` + ); + + // everything is default + let refreshMode = 'NO_REFRESH'; + let msg = 'Thank you for submitting the form!'; + let desc = ''; + + // response will not include form object if everything is default + // + if (vData.metadata?.form) { + if (vData.metadata.form?.refreshAfterSubmit) + refreshMode = vData.metadata.form.refreshAfterSubmit; + if (vData.metadata.form?.afterSubmitMessage) + msg = vData.metadata.form.afterSubmitMessage; + if (vData.metadata.form?.description) + desc = vData.metadata.form.description; + } + + const formData = { + title: viewName, + heading: viewName, + subheading: desc, + success_msg: msg, + submit_another_form: refreshMode.includes('REFRESH_BUTTON'), + show_blank_form: refreshMode.includes('AUTO_REFRESH'), + }; + + logDetailed(`NC API dbView.formCreate :: ${viewName}`); + const _perfStart = recordPerfStart(); + // const f = await api.dbView.formCreate(tblId, formData); + const f = await formViewService.formViewCreate({ + tableId: tblId, + body: formData, + }); + recordPerfStats(_perfStart, 'dbView.formCreate'); + + logDetailed( + `[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${ + formViews.length + }] Create ${viewName}` + ); + + await updateNcTblSchemaById(tblId); + + logDetailed(` Configure show/hide columns`); + await nc_configureFields( + f.id, + vData, + aTblSchema[idx].name, + viewName, + 'form' + ); + } + } + } + + async function nocoConfigureGridView(sDB, aTblSchema) { + for (let idx = 0; idx < aTblSchema.length; idx++) { + const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + const gridViews = aTblSchema[idx].views.filter((x) => x.type === 'grid'); + + let viewCnt = idx; + if (syncDB.options.syncViews) + viewCnt = rtc.view.grid + rtc.view.gallery + rtc.view.form; + rtc.view.grid += gridViews.length; + + for (let i = 0; i < (sDB.options.syncViews ? gridViews.length : 1); i++) { + logDetailed(` Axios fetch view-data`); + // fetch viewData JSON + const vData = await getViewData(gridViews[i].id); + + // retrieve view name & associated NC-ID + const viewName = aTblSchema[idx].views.find( + (x) => x.id === gridViews[i].id + )?.name; + const _perfStart = recordPerfStart(); + // const viewList: any = await api.dbView.list(tblId); + const viewList = { list: [] }; + viewList['list'] = await viewService.viewList({ + tableId: tblId, + user: { roles: userRole }, + }); + recordPerfStats(_perfStart, 'dbView.list'); + + let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id; + + logBasic( + `:: [${viewCnt + i + 1}/${rtc.view.total}] Grid : ${ + aTblSchema[idx].name + } / ${viewName}` + ); + + // create view (default already created) + if (i > 0) { + logDetailed(`NC API dbView.gridCreate :: ${viewName}`); + const _perfStart = recordPerfStart(); + const viewCreated = await gridViewService.gridViewCreate({ + tableId: tblId, + grid: { + title: viewName, + }, + }); + recordPerfStats(_perfStart, 'dbView.gridCreate'); + + await updateNcTblSchemaById(tblId); + await sMap.addToMappingTbl( + gridViews[i].id, + viewCreated.id, + viewName, + tblId + ); + ncViewId = viewCreated.id; + } + + logDetailed(` Configure show/hide columns`); + await nc_configureFields( + ncViewId, + vData, + aTblSchema[idx].name, + viewName, + 'grid' + ); + + // configure filters + if (vData?.filters) { + logDetailed(` Configure filter set`); + + // skip filters if nested + if (!vData.filters.filterSet.find((x) => x?.type === 'nested')) { + await nc_configureFilters(ncViewId, vData.filters); + } + } + + // configure sort + if (vData?.lastSortsApplied?.sortSet.length) { + logDetailed(` Configure sort set`); + await nc_configureSort(ncViewId, vData.lastSortsApplied); + } + } + } + } + + async function nocoAddUsers(aTblSchema) { + const userRoles = { + owner: 'owner', + create: 'creator', + edit: 'editor', + comment: 'commenter', + read: 'viewer', + none: 'viewer', + }; + const userList = aTblSchema.appBlanket.userInfoById; + const totalUsers = Object.keys(userList).length; + let cnt = 0; + const insertJobs: Promise[] = []; + + for (const [, value] of Object.entries( + userList as { [key: string]: any } + )) { + logDetailed( + `[${++cnt}/${totalUsers}] NC API auth.projectUserAdd :: ${value.email}` + ); + const _perfStart = recordPerfStart(); + insertJobs.push( + projectUserService + .userInvite({ + projectId: ncCreatedProjectSchema.id, + projectUser: { + email: value.email, + roles: userRoles[value.permissionLevel], + }, + req: { user: syncDB.user, clientIp: '' }, + }) + .catch((e) => + e.response?.data?.msg + ? logBasic(`NOTICE: ${e.response.data.msg}`) + : console.log(e) + ) + ); + recordPerfStats(_perfStart, 'auth.projectUserAdd'); + } + await Promise.all(insertJobs); + } + + function updateNcTblSchema(tblSchema) { + const tblId = tblSchema.id; + + // replace entry from array if already exists + const idx = ncSchema.tables.findIndex((x) => x.id === tblId); + if (idx !== -1) ncSchema.tables.splice(idx, 1); + ncSchema.tables.push(tblSchema); + + // overwrite object if it exists + ncSchema.tablesById[tblId] = tblSchema; + } + + async function updateNcTblSchemaById(tblId) { + const _perfStart = recordPerfStart(); + const ncTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: tblId, + user: syncDB.user, + }); + recordPerfStats(_perfStart, 'dbTable.read'); + + updateNcTblSchema(ncTbl); + } + + /////////////////////// + + // statistics + // + const migrationStats = []; + + async function generateMigrationStats(aTblSchema) { + const migrationStatsObj = { + table_name: '', + aTbl: { + columns: 0, + links: 0, + lookup: 0, + rollup: 0, + }, + nc: { + columns: 0, + links: 0, + lookup: 0, + rollup: 0, + invalidColumn: 0, + }, + }; + for (let idx = 0; idx < aTblSchema.length; idx++) { + migrationStatsObj.table_name = aTblSchema[idx].name; + + const aTblLinkColumns = aTblSchema[idx].columns.filter( + (x) => x.type === 'foreignKey' + ); + const aTblLookup = aTblSchema[idx].columns.filter( + (x) => x.type === 'lookup' + ); + const aTblRollup = aTblSchema[idx].columns.filter( + (x) => x.type === 'rollup' + ); + + let invalidColumnId = 0; + for (let i = 0; i < aTblLookup.length; i++) { + if ( + aTblLookup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + invalidColumnId++; + } + } + for (let i = 0; i < aTblRollup.length; i++) { + if ( + aTblRollup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + invalidColumnId++; + } + } + + migrationStatsObj.aTbl.columns = aTblSchema[idx].columns.length; + migrationStatsObj.aTbl.links = aTblLinkColumns.length; + migrationStatsObj.aTbl.lookup = aTblLookup.length; + migrationStatsObj.aTbl.rollup = aTblRollup.length; + + const ncTbl = await nc_getTableSchema(aTblSchema[idx].name); + const linkColumn = ncTbl.columns.filter( + (x) => x.uidt === UITypes.LinkToAnotherRecord + ); + const lookup = ncTbl.columns.filter((x) => x.uidt === UITypes.Lookup); + const rollup = ncTbl.columns.filter((x) => x.uidt === UITypes.Rollup); + + // all links hardwired as m2m. m2m generates additional tables per link + // hence link/2 + migrationStatsObj.nc.columns = + ncTbl.columns.length - linkColumn.length / 2; + migrationStatsObj.nc.links = linkColumn.length / 2; + migrationStatsObj.nc.lookup = lookup.length; + migrationStatsObj.nc.rollup = rollup.length; + migrationStatsObj.nc.invalidColumn = invalidColumnId; + + const temp = JSON.parse(JSON.stringify(migrationStatsObj)); + migrationStats.push(temp); + } + + const columnSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.columns; + }, 0); + const linkSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.links; + }, 0); + const lookupSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.lookup; + }, 0); + const rollupSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.rollup; + }, 0); + + logBasic(`Quick Summary:`); + logBasic(`:: Total Tables: ${aTblSchema.length}`); + logBasic(`:: Total Columns: ${columnSum}`); + logBasic(`:: Links: ${linkSum}`); + logBasic(`:: Lookup: ${lookupSum}`); + logBasic(`:: Rollup: ${rollupSum}`); + logBasic(`:: Total Filters: ${rtc.filter}`); + logBasic(`:: Total Sort: ${rtc.sort}`); + logBasic(`:: Total Views: ${rtc.view.total}`); + logBasic(`:: Grid: ${rtc.view.grid}`); + logBasic(`:: Gallery: ${rtc.view.gallery}`); + logBasic(`:: Form: ${rtc.view.form}`); + logBasic(`:: Total Records: ${rtc.data.records}`); + logBasic(`:: Total Nested Links: ${rtc.data.nestedLinks}`); + + const duration = Date.now() - start; + logBasic(`:: Migration time: ${duration}`); + logBasic(`:: Axios fetch count: ${rtc.fetchAt.count}`); + logBasic(`:: Axios fetch time: ${rtc.fetchAt.time}`); + + if (debugMode) { + await writeJsonFileAsync('stats.json', perfStats, { spaces: 2 }); + const perflog = []; + for (let i = 0; i < perfStats.length; i++) { + perflog.push(`${perfStats[i].e}, ${perfStats[i].d}`); + } + await writeJsonFileAsync('stats.csv', perflog, { spaces: 2 }); + await writeJsonFileAsync('skip.txt', rtc.migrationSkipLog.log, { + spaces: 2, + }); + } + + T.event({ + event: 'a:airtable-import:success', + data: { + stats: { + migrationTime: duration, + totalTables: aTblSchema.length, + totalColumns: columnSum, + links: linkSum, + lookup: lookupSum, + rollup: rollupSum, + totalFilters: rtc.filter, + totalSort: rtc.sort, + view: { + total: rtc.view.total, + grid: rtc.view.grid, + gallery: rtc.view.gallery, + form: rtc.view.form, + }, + axios: { + count: rtc.fetchAt.count, + time: rtc.fetchAt.time, + }, + totalRecords: rtc.data.records, + nestedLinks: rtc.data.nestedLinks, + }, + }, + }); + } + + ////////////////////////////// + // filters + + const filterMap = { + '=': 'eq', + '!=': 'neq', + '<': 'lt', + '<=': 'lte', + '>': 'gt', + '>=': 'gte', + isEmpty: 'empty', + isNotEmpty: 'notempty', + contains: 'like', + doesNotContain: 'nlike', + isAnyOf: 'anyof', + isNoneOf: 'nanyof', + '|': 'anyof', + '&': 'allof', + }; + + async function nc_configureFilters(viewId, f) { + for (let i = 0; i < f.filterSet.length; i++) { + const filter = f.filterSet[i]; + const colSchema = await nc_getColumnSchema(filter.columnId); + + // column not available; + // one of not migrated column; + if (!colSchema) { + updateMigrationSkipLog( + await sMap.getNcNameFromAtId(viewId), + colSchema.title, + colSchema.uidt, + `filter config skipped; column not migrated` + ); + continue; + } + const columnId = colSchema.id; + const datatype = colSchema.uidt; + const ncFilters = []; + + // console.log(filter) + if (datatype === UITypes.Date || datatype === UITypes.DateTime) { + // skip filters over data datatype + updateMigrationSkipLog( + await sMap.getNcNameFromAtId(viewId), + colSchema.title, + colSchema.uidt, + `filter config skipped; filter over date datatype not supported` + ); + continue; + } + + // single-select & multi-select + else if ( + datatype === UITypes.SingleSelect || + datatype === UITypes.MultiSelect + ) { + if (filter.operator === 'doesNotContain') { + filter.operator = 'isNoneOf'; + } + // if array, break it down to multiple filters + if (Array.isArray(filter.value)) { + const fx = { + fk_column_id: columnId, + logical_op: f.conjunction, + comparison_op: filterMap[filter.operator], + value: ( + await Promise.all( + filter.value.map(async (f) => await sMap.getNcNameFromAtId(f)) + ) + ).join(','), + }; + ncFilters.push(fx); + } + // not array - add as is + else if (filter.value) { + const fx = { + fk_column_id: columnId, + logical_op: f.conjunction, + comparison_op: filterMap[filter.operator], + value: await sMap.getNcNameFromAtId(filter.value), + }; + ncFilters.push(fx); + } + } + + // other data types (number/ text/ long text/ ..) + else if (filter.value) { + const fx = { + fk_column_id: columnId, + logical_op: f.conjunction, + comparison_op: filterMap[filter.operator], + value: filter.value, + }; + ncFilters.push(fx); + } + + // insert filters + for (let i = 0; i < ncFilters.length; i++) { + const _perfStart = recordPerfStart(); + await filterService.filterCreate({ + viewId: viewId, + filter: ncFilters[i], + }); + recordPerfStats(_perfStart, 'dbTableFilter.create'); + + rtc.filter++; + } + } + } + + async function nc_configureSort(viewId, s) { + for (let i = 0; i < s.sortSet.length; i++) { + const columnId = (await nc_getColumnSchema(s.sortSet[i].columnId))?.id; + + if (columnId) { + const _perfStart = recordPerfStart(); + await sortService.sortCreate({ + viewId: viewId, + sort: { + fk_column_id: columnId, + direction: s.sortSet[i].ascending ? 'asc' : 'desc', + }, + }); + recordPerfStats(_perfStart, 'dbTableSort.create'); + } + rtc.sort++; + } + } + + async function nc_configureFields(_viewId, _c, tblName, viewName, viewType?) { + // force hide PK column + const hiddenColumns = [ncSysFields.id, ncSysFields.hash]; + const c = _c.columnOrder; + + // column order corrections + // retrieve table schema + const ncTbl = await nc_getTableSchema(tblName); + // retrieve view ID + const viewId = ncTbl.views.find((x) => x.title === viewName).id; + let viewDetails; + + const _perfStart = recordPerfStart(); + if (viewType === 'form') { + viewDetails = (await formViewService.formViewGet({ formViewId: viewId })) + .columns; + recordPerfStats(_perfStart, 'dbView.formRead'); + } else if (viewType === 'gallery') { + viewDetails = ( + await galleryViewService.galleryViewGet({ + galleryViewId: viewId, + }) + ).columns; + recordPerfStats(_perfStart, 'dbView.galleryRead'); + } else { + viewDetails = await viewColumnService.columnList({ viewId: viewId }); + recordPerfStats(_perfStart, 'dbView.gridColumnsList'); + } + + // nc-specific columns; default hide. + for (let j = 0; j < hiddenColumns.length; j++) { + const ncColumnId = ncTbl.columns.find( + (x) => x.title === hiddenColumns[j] + ).id; + const ncViewColumnId = viewDetails.find( + (x) => x.fk_column_id === ncColumnId + )?.id; + if (ncViewColumnId === undefined) continue; + + // first two positions held by record id & record hash + const _perfStart = recordPerfStart(); + await viewColumnService.columnUpdate({ + viewId: viewId, + columnId: ncViewColumnId, + column: { + show: false, + order: j + 1 + c.length, + }, + }); + recordPerfStats(_perfStart, 'dbViewColumn.update'); + } + + // rest of the columns from airtable- retain order & visibility property + for (let j = 0; j < c.length; j++) { + const ncColumnId = await sMap.getNcIdFromAtId(c[j].columnId); + const ncViewColumnId = await nc_getViewColumnId( + viewId, + viewType, + ncColumnId + ); + if (ncViewColumnId === undefined) continue; + + // first two positions held by record id & record hash + const configData = { show: c[j].visibility, order: j + 1 }; + if (viewType === 'form') { + if (_c?.metadata?.form?.fieldsByColumnId?.[c[j].columnId]) { + const x = _c.metadata.form.fieldsByColumnId[c[j].columnId]; + const formData = { ...configData }; + if (x?.title) formData[`label`] = x.title; + if (x?.required) formData[`required`] = x.required; + if (x?.description) formData[`description`] = x.description; + const _perfStart = recordPerfStart(); + await formViewColumnService.columnUpdate({ + formViewColumnId: ncViewColumnId, + formViewColumn: formData, + }); + recordPerfStats(_perfStart, 'dbView.formColumnUpdate'); + } + } + const _perfStart = recordPerfStart(); + await viewColumnService.columnUpdate({ + viewId: viewId, + columnId: ncViewColumnId, + column: configData, + }); + recordPerfStats(_perfStart, 'dbViewColumn.update'); + } + } + + /////////////////////////////////////////////////////////////////////////////// + let recordCnt = 0; + try { + logBasic('SDK initialized'); + logDetailed('Project initialization started'); + // delete project if already exists + if (debugMode) await init(syncDB); + + logDetailed('Project initialized'); + + logBasic('Retrieving Airtable schema'); + // read schema file + const schema = await getAirtableSchema(syncDB); + const aTblSchema = schema.tableSchemas; + logDetailed('Project schema extraction completed'); + + if (!syncDB.projectId) { + if (!syncDB.projectName) + throw new Error('Project name or id not provided'); + // create empty project + await nocoCreateProject(syncDB.projectName); + logDetailed('Project created'); + } else { + await nocoGetProject(syncDB.projectId); + syncDB.projectName = ncCreatedProjectSchema?.title; + syncDB.baseId = syncDB.baseId || ncCreatedProjectSchema.bases[0].id; + logDetailed('Getting existing project meta'); + } + + logBasic('Importing Tables...'); + // prepare table schema (base) + await nocoCreateBaseSchema(aTblSchema); + logDetailed('Table creation completed'); + + logDetailed('Configuring Links'); + // add LTAR + await nocoCreateLinkToAnotherRecord(aTblSchema); + logDetailed('Migrating LTAR columns completed'); + + if (syncDB.options.syncLookup) { + logDetailed(`Configuring Lookup`); + // add look-ups + await nocoCreateLookups(aTblSchema); + logDetailed('Migrating Lookup columns completed'); + } + + if (syncDB.options.syncRollup) { + logDetailed('Configuring Rollup'); + // add roll-ups + await nocoCreateRollup(aTblSchema); + logDetailed('Migrating Rollup columns completed'); + + if (syncDB.options.syncLookup) { + logDetailed('Migrating Lookup form Rollup columns'); + // lookups for rollup + await nocoLookupForRollup(); + logDetailed('Migrating Lookup form Rollup columns completed'); + } + } + logDetailed('Configuring Display Value column'); + // configure Display Value + await nocoSetPrimary(aTblSchema); + logDetailed('Configuring Display Value column completed'); + + logBasic('Configuring User(s)'); + // add users + await nocoAddUsers(schema); + logDetailed('Adding users completed'); + + // hide-fields + // await nocoReconfigureFields(aTblSchema); + + logBasic('Syncing views'); + // configure views + await nocoConfigureGridView(syncDB, aTblSchema); + await nocoConfigureFormView(syncDB, aTblSchema); + await nocoConfigureGalleryView(syncDB, aTblSchema); + logDetailed('Syncing views completed'); + + if (syncDB.options.syncData) { + try { + // await nc_DumpTableSchema(); + const _perfStart = recordPerfStart(); + const ncTblList = { list: [] }; + ncTblList['list'] = await tableService.getAccessibleTables({ + projectId: ncCreatedProjectSchema.id, + baseId: syncDB.baseId, + roles: userRole, + }); + recordPerfStats(_perfStart, 'base.tableList'); + + logBasic('Reading Records...'); + + const recordsMap = {}; + + for (let i = 0; i < ncTblList.list.length; i++) { + // not a migrated table, skip + if ( + undefined === + aTblSchema.find((x) => x.name === ncTblList.list[i].title) + ) + continue; + + const _perfStart = recordPerfStart(); + const ncTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: ncTblList.list[i].id, + user: syncDB.user, + }); + recordPerfStats(_perfStart, 'dbTable.read'); + + recordCnt = 0; + + recordsMap[ncTbl.id] = await importData({ + projectName: syncDB.projectName, + table: ncTbl, + base, + logBasic, + nocoBaseDataProcessing_v2, + sDB: syncDB, + logDetailed, + }); + rtc.data.records += await recordsMap[ncTbl.id].getCount(); + + logDetailed(`Data inserted from ${ncTbl.title}`); + } + + logBasic('Configuring Record Links...'); + for (let i = 0; i < ncTblList.list.length; i++) { + // not a migrated table, skip + if ( + undefined === + aTblSchema.find((x) => x.name === ncTblList.list[i].title) + ) + continue; + + // const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + const ncTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: ncTblList.list[i].id, + user: syncDB.user, + }); + + rtc.data.nestedLinks += await importLTARData({ + table: ncTbl, + projectName: syncDB.projectName, + base, + fields: null, //Object.values(tblLinkGroup).flat(), + logBasic, + insertedAssocRef, + logDetailed, + records: recordsMap[ncTbl.id], + atNcAliasRef, + ncLinkMappingTable, + syncDB, + }); + } + } catch (error) { + logDetailed( + `There was an error while migrating data! Please make sure your API key (${syncDB.apiKey}) is correct.` + ); + logDetailed(`Error: ${error}`); + } + } + if (generate_migrationStats) { + await generateMigrationStats(aTblSchema); + } + } catch (e) { + if (e.response?.data?.msg) { + T.event({ + event: 'a:airtable-import:error', + data: { error: e.response.data.msg }, + }); + throw new Error(e.response.data.msg); + } + throw e; + } +}; + +export function getUniqueNameGenerator(defaultName = 'name') { + const namesRef = {}; + + return (initName: string = defaultName): string => { + let name = initName === '_' ? defaultName : initName; + let c = 0; + while (name in namesRef) { + name = `${initName}_${++c}`; + } + namesRef[name] = true; + return name; + }; +} + +export interface AirtableSyncConfig { + id: string; + baseURL: string; + authToken: string; + projectName?: string; + projectId?: string; + baseId?: string; + apiKey: string; + shareId: string; + user: UserType; + options: { + syncViews: boolean; + syncData: boolean; + syncRollup: boolean; + syncLookup: boolean; + syncFormula: boolean; + syncAttachment: boolean; + }; +} diff --git a/packages/nocodb-nest/src/modules/sync/helpers/readAndProcessData.ts b/packages/nocodb-nest/src/modules/sync/helpers/readAndProcessData.ts new file mode 100644 index 0000000000..3698808d02 --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/helpers/readAndProcessData.ts @@ -0,0 +1,346 @@ +import { RelationTypes, UITypes } from 'nocodb-sdk'; +import { bulkDataService, tableService } from '../..'; +import EntityMap from './EntityMap'; +import type { AirtableBase } from 'airtable/lib/airtable_base'; +import type { TableType } from 'nocodb-sdk'; + +const BULK_DATA_BATCH_SIZE = 500; +const ASSOC_BULK_DATA_BATCH_SIZE = 1000; +const BULK_PARALLEL_PROCESS = 5; + +async function readAllData({ + table, + fields, + base, + logBasic = (_str) => {}, +}: { + table: { title?: string }; + fields?; + base: AirtableBase; + logBasic?: (string) => void; + logDetailed?: (string) => void; +}): Promise { + return new Promise((resolve, reject) => { + let data = null; + + const selectParams: any = { + pageSize: 100, + }; + + if (fields) selectParams.fields = fields; + + base(table.title) + .select(selectParams) + .eachPage( + async function page(records, fetchNextPage) { + if (!data) { + data = new EntityMap(); + await data.init(); + } + + for await (const record of records) { + await data.addRow({ id: record.id, ...record.fields }); + } + + const tmpLength = await data.getCount(); + + logBasic( + `:: Reading '${table.title}' data :: ${Math.max( + 1, + tmpLength - records.length, + )} - ${tmpLength}`, + ); + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + async function done(err) { + if (err) { + console.error(err); + return reject(err); + } + resolve(data); + }, + ); + }); +} + +export async function importData({ + projectName, + table, + base, + nocoBaseDataProcessing_v2, + sDB, + logDetailed = (_str) => {}, + logBasic = (_str) => {}, +}: { + projectName: string; + table: { title?: string; id?: string }; + fields?; + base: AirtableBase; + logBasic: (string) => void; + logDetailed: (string) => void; + nocoBaseDataProcessing_v2; + sDB; +}): Promise { + try { + // @ts-ignore + const records = await readAllData({ + table, + base, + logDetailed, + logBasic, + }); + + await new Promise(async (resolve) => { + const readable = records.getStream(); + const allRecordsCount = await records.getCount(); + const promises = []; + let tempData = []; + let importedCount = 0; + let activeProcess = 0; + readable.on('data', async (record) => { + promises.push( + new Promise(async (resolve) => { + activeProcess++; + if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause(); + const { id: rid, ...fields } = record; + const r = await nocoBaseDataProcessing_v2(sDB, table, { + id: rid, + fields, + }); + tempData.push(r); + + if (tempData.length >= BULK_DATA_BATCH_SIZE) { + let insertArray = tempData.splice(0, tempData.length); + + await bulkDataService.bulkDataInsert({ + projectName, + tableName: table.title, + body: insertArray, + cookie: {}, + }); + + logBasic( + `:: Importing '${ + table.title + }' data :: ${importedCount} - ${Math.min( + importedCount + BULK_DATA_BATCH_SIZE, + allRecordsCount, + )}`, + ); + importedCount += insertArray.length; + insertArray = []; + } + activeProcess--; + if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume(); + resolve(true); + }), + ); + }); + readable.on('end', async () => { + await Promise.all(promises); + if (tempData.length > 0) { + await bulkDataService.bulkDataInsert({ + projectName, + tableName: table.title, + body: tempData, + cookie: {}, + }); + + logBasic( + `:: Importing '${ + table.title + }' data :: ${importedCount} - ${Math.min( + importedCount + BULK_DATA_BATCH_SIZE, + allRecordsCount, + )}`, + ); + importedCount += tempData.length; + tempData = []; + } + resolve(true); + }); + }); + + return records; + } catch (e) { + console.log(e); + return null; + } +} + +export async function importLTARData({ + table, + fields, + base, + projectName, + insertedAssocRef = {}, + logDetailed = (_str) => {}, + logBasic = (_str) => {}, + records, + atNcAliasRef, + ncLinkMappingTable, + syncDB, +}: { + projectName: string; + table: { title?: string; id?: string }; + fields; + base: AirtableBase; + logDetailed: (string) => void; + logBasic: (string) => void; + insertedAssocRef: { [assocTableId: string]: boolean }; + records?: EntityMap; + atNcAliasRef: { + [ncTableId: string]: { + [ncTitle: string]: string; + }; + }; + ncLinkMappingTable: Record>[]; + syncDB; +}) { + const assocTableMetas: Array<{ + modelMeta: { id?: string; title?: string }; + colMeta: { title?: string }; + curCol: { title?: string }; + refCol: { title?: string }; + }> = []; + const allData = + records || + (await readAllData({ + table, + fields, + base, + logDetailed, + logBasic, + })); + + const modelMeta: any = await tableService.getTableWithAccessibleViews({ + tableId: table.id, + user: syncDB.user, + }); + + for (const colMeta of modelMeta.columns) { + // skip columns which are not LTAR and Many to many + if ( + colMeta.uidt !== UITypes.LinkToAnotherRecord || + colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY + ) { + continue; + } + + // skip if already inserted + if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue; + + // self links: skip if the column under consideration is the add-on column NocoDB creates + if (ncLinkMappingTable.every((a) => a.nc.title !== colMeta.title)) continue; + + // mark as inserted + insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; + + const assocModelMeta: TableType = + (await tableService.getTableWithAccessibleViews({ + tableId: colMeta.colOptions.fk_mm_model_id, + user: syncDB.user, + })) as any; + + // extract associative table and columns meta + assocTableMetas.push({ + modelMeta: assocModelMeta, + colMeta, + curCol: assocModelMeta.columns.find( + (c) => c.id === colMeta.colOptions.fk_mm_child_column_id, + ), + refCol: assocModelMeta.columns.find( + (c) => c.id === colMeta.colOptions.fk_mm_parent_column_id, + ), + }); + } + + let nestedLinkCnt = 0; + // Iterate over all related M2M associative table + for await (const assocMeta of assocTableMetas) { + let assocTableData = []; + let importedCount = 0; + + // extract insert data from records + await new Promise((resolve) => { + const promises = []; + const readable = allData.getStream(); + let activeProcess = 0; + readable.on('data', async (record) => { + promises.push( + new Promise(async (resolve) => { + activeProcess++; + if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause(); + const { id: _atId, ...rec } = record; + + // todo: use actual alias instead of sanitized + assocTableData.push( + ...( + rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || [] + ).map((id) => ({ + [assocMeta.curCol.title]: record.id, + [assocMeta.refCol.title]: id, + })), + ); + + if (assocTableData.length >= ASSOC_BULK_DATA_BATCH_SIZE) { + let insertArray = assocTableData.splice(0, assocTableData.length); + logBasic( + `:: Importing '${ + table.title + }' LTAR data :: ${importedCount} - ${Math.min( + importedCount + ASSOC_BULK_DATA_BATCH_SIZE, + insertArray.length, + )}`, + ); + + await bulkDataService.bulkDataInsert({ + projectName, + tableName: assocMeta.modelMeta.title, + body: insertArray, + cookie: {}, + }); + + importedCount += insertArray.length; + insertArray = []; + } + activeProcess--; + if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume(); + resolve(true); + }), + ); + }); + readable.on('end', async () => { + await Promise.all(promises); + if (assocTableData.length >= 0) { + logBasic( + `:: Importing '${ + table.title + }' LTAR data :: ${importedCount} - ${Math.min( + importedCount + ASSOC_BULK_DATA_BATCH_SIZE, + assocTableData.length, + )}`, + ); + + await bulkDataService.bulkDataInsert({ + projectName, + tableName: assocMeta.modelMeta.title, + body: assocTableData, + cookie: {}, + }); + + importedCount += assocTableData.length; + assocTableData = []; + } + resolve(true); + }); + }); + + nestedLinkCnt += importedCount; + } + return nestedLinkCnt; +} diff --git a/packages/nocodb-nest/src/modules/sync/helpers/syncMap.ts b/packages/nocodb-nest/src/modules/sync/helpers/syncMap.ts new file mode 100644 index 0000000000..7855bbaa2c --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/helpers/syncMap.ts @@ -0,0 +1,31 @@ +export const mapTbl = {}; + +// static mapping records between aTblId && ncId +export const addToMappingTbl = function addToMappingTbl( + aTblId, + ncId, + ncName, + parent? +) { + mapTbl[aTblId] = { + ncId: ncId, + ncParent: parent, + // name added to assist in quick debug + ncName: ncName, + }; +}; + +// get NcID from airtable ID +export const getNcIdFromAtId = function getNcIdFromAtId(aId) { + return mapTbl[aId]?.ncId; +}; + +// get nc Parent from airtable ID +export const getNcParentFromAtId = function getNcParentFromAtId(aId) { + return mapTbl[aId]?.ncParent; +}; + +// get nc-title from airtable ID +export const getNcNameFromAtId = function getNcNameFromAtId(aId) { + return mapTbl[aId]?.ncName; +}; diff --git a/packages/nocodb-nest/src/modules/sync/sync.controller.spec.ts b/packages/nocodb-nest/src/modules/sync/sync.controller.spec.ts new file mode 100644 index 0000000000..de758dd9a7 --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/sync.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { SyncController } from './sync.controller'; +import { SyncService } from './sync.service'; + +describe('SyncController', () => { + let controller: SyncController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [SyncController], + providers: [SyncService], + }).compile(); + + controller = module.get(SyncController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/sync/sync.controller.ts b/packages/nocodb-nest/src/modules/sync/sync.controller.ts new file mode 100644 index 0000000000..aa5c75aba3 --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/sync.controller.ts @@ -0,0 +1,67 @@ +import { + Body, + Controller, + Delete, + Get, + Param, + Patch, + Post, + Req, +} from '@nestjs/common'; +import { Acl } from '../../middlewares/extract-project-id/extract-project-id.middleware'; +import { SyncService } from './sync.service'; + +@Controller('sync') +export class SyncController { + constructor(private readonly syncService: SyncService) {} + + @Get([ + '/api/v1/db/meta/projects/:projectId/syncs', + '/api/v1/db/meta/projects/:projectId/syncs/:baseId', + ]) + @Acl('syncSourceList') + async syncSourceList( + @Param('projectId') projectId: string, + @Param('baseId') baseId?: string, + ) { + return await this.syncService.syncSourceList({ + projectId, + baseId, + }); + } + + @Post([ + '/api/v1/db/meta/projects/:projectId/syncs', + '/api/v1/db/meta/projects/:projectId/syncs/:baseId', + ]) + @Acl('syncSourceCreate') + async syncCreate( + @Param('projectId') projectId: string, + @Body() body: any, + @Req() req, + @Param('baseId') baseId?: string, + ) { + return await this.syncService.syncCreate({ + projectId: projectId, + baseId: baseId, + userId: (req as any).user.id, + syncPayload: body, + }); + } + + @Delete('/api/v1/db/meta/syncs/:syncId') + @Acl('syncSourceDelete') + async syncDelete(@Param('syncId') syncId: string) { + return await this.syncService.syncDelete({ + syncId: syncId, + }); + } + + @Patch('/api/v1/db/meta/syncs/:syncId') + async syncUpdate(@Param('syncId') syncId: string, @Body() body: any) { + return await this.syncService.syncUpdate({ + syncId: syncId, + syncPayload: body, + }); + } +} diff --git a/packages/nocodb-nest/src/modules/sync/sync.module.ts b/packages/nocodb-nest/src/modules/sync/sync.module.ts new file mode 100644 index 0000000000..2bd664e7d2 --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/sync.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { SyncService } from './sync.service'; +import { SyncController } from './sync.controller'; + +@Module({ + controllers: [SyncController], + providers: [SyncService] +}) +export class SyncModule {} diff --git a/packages/nocodb-nest/src/modules/sync/sync.service.spec.ts b/packages/nocodb-nest/src/modules/sync/sync.service.spec.ts new file mode 100644 index 0000000000..fd1d96394a --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/sync.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { SyncService } from './sync.service'; + +describe('SyncService', () => { + let service: SyncService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [SyncService], + }).compile(); + + service = module.get(SyncService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/sync/sync.service.ts b/packages/nocodb-nest/src/modules/sync/sync.service.ts new file mode 100644 index 0000000000..c573a34b33 --- /dev/null +++ b/packages/nocodb-nest/src/modules/sync/sync.service.ts @@ -0,0 +1,45 @@ +import { Injectable } from '@nestjs/common'; +import { PagedResponseImpl } from '../../helpers/PagedResponse'; +import { Project, SyncSource } from '../../models'; +import { T } from 'nc-help'; + +@Injectable() +export class SyncService { + async syncSourceList(param: { projectId: string; baseId?: string }) { + return new PagedResponseImpl( + await SyncSource.list(param.projectId, param.baseId), + ); + } + + async syncCreate(param: { + projectId: string; + baseId?: string; + userId: string; + syncPayload: Partial; + }) { + T.emit('evt', { evt_type: 'syncSource:created' }); + const project = await Project.getWithInfo(param.projectId); + + const sync = await SyncSource.insert({ + ...param.syncPayload, + fk_user_id: param.userId, + base_id: param.baseId ? param.baseId : project.bases[0].id, + project_id: param.projectId, + }); + return sync; + } + + async syncDelete(param: { syncId: string }) { + T.emit('evt', { evt_type: 'syncSource:deleted' }); + return await SyncSource.delete(param.syncId); + } + + async syncUpdate(param: { + syncId: string; + syncPayload: Partial; + }) { + T.emit('evt', { evt_type: 'syncSource:updated' }); + + return await SyncSource.update(param.syncId, param.syncPayload); + } +} diff --git a/packages/nocodb-nest/src/modules/test/test.controller.spec.ts b/packages/nocodb-nest/src/modules/test/test.controller.spec.ts new file mode 100644 index 0000000000..c3bc16d6b7 --- /dev/null +++ b/packages/nocodb-nest/src/modules/test/test.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { TestController } from './test.controller'; +import { TestService } from './test.service'; + +describe('TestController', () => { + let controller: TestController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [TestController], + providers: [TestService], + }).compile(); + + controller = module.get(TestController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/test/test.controller.ts b/packages/nocodb-nest/src/modules/test/test.controller.ts new file mode 100644 index 0000000000..99725a622d --- /dev/null +++ b/packages/nocodb-nest/src/modules/test/test.controller.ts @@ -0,0 +1,7 @@ +import { Controller } from '@nestjs/common'; +import { TestService } from './test.service'; + +@Controller('test') +export class TestController { + constructor(private readonly testService: TestService) {} +} diff --git a/packages/nocodb-nest/src/modules/test/test.module.ts b/packages/nocodb-nest/src/modules/test/test.module.ts new file mode 100644 index 0000000000..04260c05da --- /dev/null +++ b/packages/nocodb-nest/src/modules/test/test.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { TestService } from './test.service'; +import { TestController } from './test.controller'; + +@Module({ + controllers: [TestController], + providers: [TestService] +}) +export class TestModule {} diff --git a/packages/nocodb-nest/src/modules/test/test.service.spec.ts b/packages/nocodb-nest/src/modules/test/test.service.spec.ts new file mode 100644 index 0000000000..1aacbe5d55 --- /dev/null +++ b/packages/nocodb-nest/src/modules/test/test.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { TestService } from './test.service'; + +describe('TestService', () => { + let service: TestService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [TestService], + }).compile(); + + service = module.get(TestService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/test/test.service.ts b/packages/nocodb-nest/src/modules/test/test.service.ts new file mode 100644 index 0000000000..0d79b70d7f --- /dev/null +++ b/packages/nocodb-nest/src/modules/test/test.service.ts @@ -0,0 +1,4 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class TestService {}