Browse Source

feat: plugi, caches apis

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/5444/head
Pranav C 2 years ago
parent
commit
34a9fe5b81
  1. 7
      packages/nocodb-nest/src/app.module.ts
  2. 20
      packages/nocodb-nest/src/modules/caches/caches.controller.spec.ts
  3. 7
      packages/nocodb-nest/src/modules/caches/caches.controller.ts
  4. 9
      packages/nocodb-nest/src/modules/caches/caches.module.ts
  5. 18
      packages/nocodb-nest/src/modules/caches/caches.service.spec.ts
  6. 4
      packages/nocodb-nest/src/modules/caches/caches.service.ts
  7. 20
      packages/nocodb-nest/src/modules/import/import.controller.spec.ts
  8. 142
      packages/nocodb-nest/src/modules/import/import.controller.ts
  9. 9
      packages/nocodb-nest/src/modules/import/import.module.ts
  10. 18
      packages/nocodb-nest/src/modules/import/import.service.spec.ts
  11. 4
      packages/nocodb-nest/src/modules/import/import.service.ts
  12. 20
      packages/nocodb-nest/src/modules/plugins/plugins.controller.spec.ts
  13. 52
      packages/nocodb-nest/src/modules/plugins/plugins.controller.ts
  14. 9
      packages/nocodb-nest/src/modules/plugins/plugins.module.ts
  15. 18
      packages/nocodb-nest/src/modules/plugins/plugins.service.spec.ts
  16. 40
      packages/nocodb-nest/src/modules/plugins/plugins.service.ts
  17. 222
      packages/nocodb-nest/src/modules/sync/helpers/EntityMap.ts
  18. 6
      packages/nocodb-nest/src/modules/sync/helpers/NocoSyncDestAdapter.ts
  19. 7
      packages/nocodb-nest/src/modules/sync/helpers/NocoSyncSourceAdapter.ts
  20. 238
      packages/nocodb-nest/src/modules/sync/helpers/fetchAT.ts
  21. 2444
      packages/nocodb-nest/src/modules/sync/helpers/job.ts
  22. 346
      packages/nocodb-nest/src/modules/sync/helpers/readAndProcessData.ts
  23. 31
      packages/nocodb-nest/src/modules/sync/helpers/syncMap.ts
  24. 20
      packages/nocodb-nest/src/modules/sync/sync.controller.spec.ts
  25. 67
      packages/nocodb-nest/src/modules/sync/sync.controller.ts
  26. 9
      packages/nocodb-nest/src/modules/sync/sync.module.ts
  27. 18
      packages/nocodb-nest/src/modules/sync/sync.service.spec.ts
  28. 45
      packages/nocodb-nest/src/modules/sync/sync.service.ts
  29. 20
      packages/nocodb-nest/src/modules/test/test.controller.spec.ts
  30. 7
      packages/nocodb-nest/src/modules/test/test.controller.ts
  31. 9
      packages/nocodb-nest/src/modules/test/test.module.ts
  32. 18
      packages/nocodb-nest/src/modules/test/test.service.spec.ts
  33. 4
      packages/nocodb-nest/src/modules/test/test.service.ts

7
packages/nocodb-nest/src/app.module.ts

@ -41,9 +41,14 @@ import { ApiDocsModule } from './modules/api-docs/api-docs.module';
import { PublicMetasModule } from './modules/public-metas/public-metas.module'; import { PublicMetasModule } from './modules/public-metas/public-metas.module';
import { PublicDatasModule } from './modules/public-datas/public-datas.module'; import { PublicDatasModule } from './modules/public-datas/public-datas.module';
import { PublicDatasExportModule } from './modules/public-datas-export/public-datas-export.module'; import { PublicDatasExportModule } from './modules/public-datas-export/public-datas-export.module';
import { SyncModule } from './modules/sync/sync.module';
import { ImportModule } from './modules/import/import.module';
import { CachesModule } from './modules/caches/caches.module';
import { TestModule } from './modules/test/test.module';
import { PluginsModule } from './modules/plugins/plugins.module';
@Module({ @Module({
imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule, AuditsModule, DatasModule, DataAliasModule, ApiDocsModule, PublicMetasModule, PublicDatasModule, PublicDatasExportModule], imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule, AuditsModule, DatasModule, DataAliasModule, ApiDocsModule, PublicMetasModule, PublicDatasModule, PublicDatasExportModule, SyncModule, ImportModule, CachesModule, TestModule, PluginsModule],
controllers: [], controllers: [],
providers: [Connection, MetaService, JwtStrategy, ExtractProjectIdMiddleware], providers: [Connection, MetaService, JwtStrategy, ExtractProjectIdMiddleware],
exports: [Connection, MetaService], exports: [Connection, MetaService],

20
packages/nocodb-nest/src/modules/caches/caches.controller.spec.ts

@ -0,0 +1,20 @@
import { Test, TestingModule } from '@nestjs/testing';
import { CachesController } from './caches.controller';
import { CachesService } from './caches.service';
describe('CachesController', () => {
let controller: CachesController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [CachesController],
providers: [CachesService],
}).compile();
controller = module.get<CachesController>(CachesController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

7
packages/nocodb-nest/src/modules/caches/caches.controller.ts

@ -0,0 +1,7 @@
import { Controller } from '@nestjs/common';
import { CachesService } from './caches.service';
@Controller('caches')
export class CachesController {
constructor(private readonly cachesService: CachesService) {}
}

9
packages/nocodb-nest/src/modules/caches/caches.module.ts

@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { CachesService } from './caches.service';
import { CachesController } from './caches.controller';
@Module({
controllers: [CachesController],
providers: [CachesService]
})
export class CachesModule {}

18
packages/nocodb-nest/src/modules/caches/caches.service.spec.ts

@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { CachesService } from './caches.service';
describe('CachesService', () => {
let service: CachesService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [CachesService],
}).compile();
service = module.get<CachesService>(CachesService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

4
packages/nocodb-nest/src/modules/caches/caches.service.ts

@ -0,0 +1,4 @@
import { Injectable } from '@nestjs/common';
@Injectable()
export class CachesService {}

20
packages/nocodb-nest/src/modules/import/import.controller.spec.ts

@ -0,0 +1,20 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ImportController } from './import.controller';
import { ImportService } from './import.service';
describe('ImportController', () => {
let controller: ImportController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [ImportController],
providers: [ImportService],
}).compile();
controller = module.get<ImportController>(ImportController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

142
packages/nocodb-nest/src/modules/import/import.controller.ts

@ -0,0 +1,142 @@
import { Controller } from '@nestjs/common';
import { ImportService } from './import.service';
const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB';
const AIRTABLE_PROGRESS_JOB = 'AIRTABLE_PROGRESS_JOB';
enum SyncStatus {
PROGRESS = 'PROGRESS',
COMPLETED = 'COMPLETED',
FAILED = 'FAILED',
}
@Controller('import')
export class ImportController {
constructor(private readonly importService: ImportService) {}
default (
router: Router,
sv: Server,
jobs: { [id: string]: { last_message: any } }
) => {
// add importer job handler and progress notification job handler
NocoJobs.jobsMgr.addJobWorker(
AIRTABLE_IMPORT_JOB,
syncService.airtableImportJob
);
NocoJobs.jobsMgr.addJobWorker(
AIRTABLE_PROGRESS_JOB,
({ payload, progress }) => {
sv.to(payload?.id).emit('progress', {
msg: progress?.msg,
level: progress?.level,
status: progress?.status,
});
if (payload?.id in jobs) {
jobs[payload?.id].last_message = {
msg: progress?.msg,
level: progress?.level,
status: progress?.status,
};
}
}
);
NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => {
NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, {
payload,
progress: {
msg: progress?.msg,
level: progress?.level,
status: progress?.status,
},
});
});
NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, (payload) => {
NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, {
payload,
progress: {
msg: 'Complete!',
status: SyncStatus.COMPLETED,
},
});
delete jobs[payload?.id];
});
NocoJobs.jobsMgr.addFailureCbk(AIRTABLE_IMPORT_JOB, (payload, error: any) => {
NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, {
payload,
progress: {
msg: error?.message || 'Failed due to some internal error',
status: SyncStatus.FAILED,
},
});
delete jobs[payload?.id];
});
router.post(
'/api/v1/db/meta/import/airtable',
catchError((req, res) => {
NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, {
id: req.query.id,
...req.body,
});
res.json({});
})
);
router.post(
'/api/v1/db/meta/syncs/:syncId/trigger',
catchError(async (req: Request, res) => {
if (req.params.syncId in jobs) {
NcError.badRequest('Sync already in progress');
}
const syncSource = await SyncSource.get(req.params.syncId);
const user = await syncSource.getUser();
const token = userService.genJwt(user, Noco.getConfig());
// Treat default baseUrl as siteUrl from req object
let baseURL = (req as any).ncSiteUrl;
// if environment value avail use it
// or if it's docker construct using `PORT`
if (process.env.NC_DOCKER) {
baseURL = `http://localhost:${process.env.PORT || 8080}`;
}
setTimeout(() => {
NocoJobs.jobsMgr.add<AirtableSyncConfig>(AIRTABLE_IMPORT_JOB, {
id: req.params.syncId,
...(syncSource?.details || {}),
projectId: syncSource.project_id,
baseId: syncSource.base_id,
authToken: token,
baseURL,
user: user,
});
}, 1000);
jobs[req.params.syncId] = {
last_message: {
msg: 'Sync started',
},
};
res.json({});
})
);
router.post(
'/api/v1/db/meta/syncs/:syncId/abort',
catchError(async (req: Request, res) => {
if (req.params.syncId in jobs) {
delete jobs[req.params.syncId];
}
res.json({});
})
);
};
}

9
packages/nocodb-nest/src/modules/import/import.module.ts

@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { ImportService } from './import.service';
import { ImportController } from './import.controller';
@Module({
controllers: [ImportController],
providers: [ImportService]
})
export class ImportModule {}

18
packages/nocodb-nest/src/modules/import/import.service.spec.ts

@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ImportService } from './import.service';
describe('ImportService', () => {
let service: ImportService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [ImportService],
}).compile();
service = module.get<ImportService>(ImportService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

4
packages/nocodb-nest/src/modules/import/import.service.ts

@ -0,0 +1,4 @@
import { Injectable } from '@nestjs/common';
@Injectable()
export class ImportService {}

20
packages/nocodb-nest/src/modules/plugins/plugins.controller.spec.ts

@ -0,0 +1,20 @@
import { Test, TestingModule } from '@nestjs/testing';
import { PluginsController } from './plugins.controller';
import { PluginsService } from './plugins.service';
describe('PluginsController', () => {
let controller: PluginsController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [PluginsController],
providers: [PluginsService],
}).compile();
controller = module.get<PluginsController>(PluginsController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

52
packages/nocodb-nest/src/modules/plugins/plugins.controller.ts

@ -0,0 +1,52 @@
import { Body, Controller, Get, Param, Patch, Post } from '@nestjs/common';
import { PagedResponseImpl } from '../../helpers/PagedResponse';
import { Acl } from '../../middlewares/extract-project-id/extract-project-id.middleware';
import { PluginsService } from './plugins.service';
// todo: move to a interceptor
const blockInCloudMw = (_req, res, next) => {
if (process.env.NC_CLOUD === 'true') {
res.status(403).send('Not allowed');
} else next();
};
@Controller('plugins')
export class PluginsController {
constructor(private readonly pluginsService: PluginsService) {}
@Get('/api/v1/db/meta/plugins')
@Acl('pluginList')
async pluginList() {
return new PagedResponseImpl(await this.pluginsService.pluginList());
}
@Post('/api/v1/db/meta/plugins/test')
@Acl('pluginTest')
async pluginTest(@Body() body: any) {
return await this.pluginsService.pluginTest({ body: body });
}
@Get('/api/v1/db/meta/plugins/:pluginId')
@Acl('pluginRead')
async pluginRead(@Param('pluginId') pluginId: string) {
return await this.pluginsService.pluginRead({ pluginId: pluginId });
}
@Patch('/api/v1/db/meta/plugins/:pluginId')
@Acl('pluginUpdate')
async pluginUpdate(@Body() body: any, @Param('pluginId') pluginId: string) {
const plugin = await this.pluginsService.pluginUpdate({
pluginId: pluginId,
plugin: body,
});
return plugin;
}
@Get('/api/v1/db/meta/plugins/:pluginTitle/status')
@Acl('isPluginActive')
async isPluginActive(@Param('pluginTitle') pluginTitle: string) {
return await this.pluginsService.isPluginActive({
pluginTitle: pluginTitle,
});
}
}

9
packages/nocodb-nest/src/modules/plugins/plugins.module.ts

@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { PluginsService } from './plugins.service';
import { PluginsController } from './plugins.controller';
@Module({
controllers: [PluginsController],
providers: [PluginsService]
})
export class PluginsModule {}

18
packages/nocodb-nest/src/modules/plugins/plugins.service.spec.ts

@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { PluginsService } from './plugins.service';
describe('PluginsService', () => {
let service: PluginsService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [PluginsService],
}).compile();
service = module.get<PluginsService>(PluginsService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

40
packages/nocodb-nest/src/modules/plugins/plugins.service.ts

@ -0,0 +1,40 @@
import { Injectable } from '@nestjs/common';
import { PluginTestReqType, PluginType } from 'nocodb-sdk';
import { validatePayload } from '../../helpers';
import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2';
import { Plugin } from '../../models';
import { T } from 'nc-help';
@Injectable()
export class PluginsService {
async pluginList() {
return await Plugin.list();
}
async pluginTest(param: { body: PluginTestReqType }) {
validatePayload(
'swagger.json#/components/schemas/PluginTestReq',
param.body,
);
T.emit('evt', { evt_type: 'plugin:tested' });
return await NcPluginMgrv2.test(param.body);
}
async pluginRead(param: { pluginId: string }) {
return await Plugin.get(param.pluginId);
}
async pluginUpdate(param: { pluginId: string; plugin: PluginType }) {
validatePayload('swagger.json#/components/schemas/PluginReq', param.plugin);
const plugin = await Plugin.update(param.pluginId, param.plugin);
T.emit('evt', {
evt_type: plugin.active ? 'plugin:installed' : 'plugin:uninstalled',
title: plugin.title,
});
return plugin;
}
async isPluginActive(param: { pluginTitle: string }) {
return await Plugin.isPluginActive(param.pluginTitle);
}
}

222
packages/nocodb-nest/src/modules/sync/helpers/EntityMap.ts

@ -0,0 +1,222 @@
import { Readable } from 'stream';
import sqlite3 from 'sqlite3';
class EntityMap {
initialized: boolean;
cols: string[];
db: any;
constructor(...args) {
this.initialized = false;
this.cols = args.map((arg) => processKey(arg));
this.db = new Promise((resolve, reject) => {
const db = new sqlite3.Database(':memory:');
const colStatement =
this.cols.length > 0
? this.cols.join(' TEXT, ') + ' TEXT'
: 'mappingPlaceholder TEXT';
db.run(`CREATE TABLE mapping (${colStatement})`, (err) => {
if (err) {
console.log(err);
reject(err);
}
resolve(db);
});
});
}
async init() {
if (!this.initialized) {
this.db = await this.db;
this.initialized = true;
}
}
destroy() {
if (this.initialized && this.db) {
this.db.close();
}
}
async addRow(row) {
if (!this.initialized) {
throw 'Please initialize first!';
}
const cols = Object.keys(row).map((key) => processKey(key));
const colStatement = cols.map((key) => `'${key}'`).join(', ');
const questionMarks = cols.map(() => '?').join(', ');
const promises = [];
for (const col of cols.filter((col) => !this.cols.includes(col))) {
promises.push(
new Promise((resolve, reject) => {
this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => {
if (err) {
console.log(err);
reject(err);
}
this.cols.push(col);
resolve(true);
});
})
);
}
await Promise.all(promises);
const values = Object.values(row).map((val) => {
if (typeof val === 'object') {
return `JSON::${JSON.stringify(val)}`;
}
return val;
});
return new Promise((resolve, reject) => {
this.db.run(
`INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`,
values,
(err) => {
if (err) {
console.log(err);
reject(err);
}
resolve(true);
}
);
});
}
getRow(col, val, res = []): Promise<Record<string, any>> {
if (!this.initialized) {
throw 'Please initialize first!';
}
return new Promise((resolve, reject) => {
col = processKey(col);
res = res.map((r) => processKey(r));
this.db.get(
`SELECT ${
res.length ? res.join(', ') : '*'
} FROM mapping WHERE ${col} = ?`,
[val],
(err, rs) => {
if (err) {
console.log(err);
reject(err);
}
if (rs) {
rs = processResponseRow(rs);
}
resolve(rs);
}
);
});
}
getCount(): Promise<number> {
if (!this.initialized) {
throw 'Please initialize first!';
}
return new Promise((resolve, reject) => {
this.db.get(`SELECT COUNT(*) as count FROM mapping`, (err, rs) => {
if (err) {
console.log(err);
reject(err);
}
resolve(rs.count);
});
});
}
getStream(res = []): DBStream {
if (!this.initialized) {
throw 'Please initialize first!';
}
res = res.map((r) => processKey(r));
return new DBStream(
this.db,
`SELECT ${res.length ? res.join(', ') : '*'} FROM mapping`
);
}
getLimit(limit, offset, res = []): Promise<Record<string, any>[]> {
if (!this.initialized) {
throw 'Please initialize first!';
}
return new Promise((resolve, reject) => {
res = res.map((r) => processKey(r));
this.db.all(
`SELECT ${
res.length ? res.join(', ') : '*'
} FROM mapping LIMIT ${limit} OFFSET ${offset}`,
(err, rs) => {
if (err) {
console.log(err);
reject(err);
}
for (let row of rs) {
row = processResponseRow(row);
}
resolve(rs);
}
);
});
}
}
class DBStream extends Readable {
db: any;
stmt: any;
sql: any;
constructor(db, sql) {
super({ objectMode: true });
this.db = db;
this.sql = sql;
this.stmt = this.db.prepare(this.sql);
this.on('end', () => this.stmt.finalize());
}
_read() {
const stream = this;
this.stmt.get(function (err, result) {
if (err) {
stream.emit('error', err);
} else {
if (result) {
result = processResponseRow(result);
}
stream.push(result || null);
}
});
}
}
function processResponseRow(res: any) {
for (const key of Object.keys(res)) {
if (res[key] && res[key].startsWith('JSON::')) {
try {
res[key] = JSON.parse(res[key].replace('JSON::', ''));
} catch (e) {
console.log(e);
}
}
if (revertKey(key) !== key) {
res[revertKey(key)] = res[key];
delete res[key];
}
}
return res;
}
function processKey(key) {
return key.replace(/'/g, "''").replace(/[A-Z]/g, (match) => `_${match}`);
}
function revertKey(key) {
return key.replace(/''/g, "'").replace(/_[A-Z]/g, (match) => match[1]);
}
export default EntityMap;

6
packages/nocodb-nest/src/modules/sync/helpers/NocoSyncDestAdapter.ts

@ -0,0 +1,6 @@
export abstract class NocoSyncSourceAdapter {
public abstract init(): Promise<void>;
public abstract destProjectWrite(): Promise<any>;
public abstract destSchemaWrite(): Promise<any>;
public abstract destDataWrite(): Promise<any>;
}

7
packages/nocodb-nest/src/modules/sync/helpers/NocoSyncSourceAdapter.ts

@ -0,0 +1,7 @@
export abstract class NocoSyncSourceAdapter {
public abstract init(): Promise<void>;
public abstract srcSchemaGet(): Promise<any>;
public abstract srcDataLoad(): Promise<any>;
public abstract srcDataListen(): Promise<any>;
public abstract srcDataPoll(): Promise<any>;
}

238
packages/nocodb-nest/src/modules/sync/helpers/fetchAT.ts

@ -0,0 +1,238 @@
import axios from 'axios';
const info: any = {
initialized: false,
};
async function initialize(shareId) {
info.cookie = '';
const url = `https://airtable.com/${shareId}`;
try {
const hreq = await axios
.get(url, {
headers: {
accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
},
referrerPolicy: 'strict-origin-when-cross-origin',
body: null,
method: 'GET',
})
.then((response) => {
for (const ck of response.headers['set-cookie']) {
info.cookie += ck.split(';')[0] + '; ';
}
return response.data;
})
.catch(() => {
throw {
message:
'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
};
});
info.headers = JSON.parse(
hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim()
);
info.link = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim());
info.baseInfo = decodeURIComponent(info.link)
.match(/{(.*)}/g)[0]
.split('&')
.reduce((result, el) => {
try {
return Object.assign(
result,
JSON.parse(el.includes('=') ? el.split('=')[1] : el)
);
} catch (e) {
if (el.includes('=')) {
return Object.assign(result, {
[el.split('=')[0]]: el.split('=')[1],
});
}
}
}, {});
info.baseId = info.baseInfo.applicationId;
info.initialized = true;
} catch (e) {
console.log(e);
info.initialized = false;
if (e.message) {
throw e;
} else {
throw {
message:
'Error processing Shared Base :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
};
}
}
}
async function read() {
if (info.initialized) {
const resreq = await axios('https://airtable.com' + info.link, {
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
referrerPolicy: 'no-referrer',
body: null,
method: 'GET',
})
.then((response) => {
return response.data;
})
.catch(() => {
throw {
message:
'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
};
});
return {
schema: resreq.data,
baseId: info.baseId,
baseInfo: info.baseInfo,
};
} else {
throw {
message: 'Error Initializing :: please try again !!',
};
}
}
async function readView(viewId) {
if (info.initialized) {
const resreq = await axios(
`https://airtable.com/v0.3/view/${viewId}/readData?` +
`stringifiedObjectParams=${encodeURIComponent('{}')}&requestId=${
info.baseInfo.requestId
}&accessPolicy=${encodeURIComponent(
JSON.stringify({
allowedActions: info.baseInfo.allowedActions,
shareId: info.baseInfo.shareId,
applicationId: info.baseInfo.applicationId,
generationNumber: info.baseInfo.generationNumber,
expires: info.baseInfo.expires,
signature: info.baseInfo.signature,
})
)}`,
{
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
referrerPolicy: 'no-referrer',
body: null,
method: 'GET',
}
)
.then((response) => {
return response.data;
})
.catch(() => {
throw {
message:
'Error Reading View :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
};
});
return { view: resreq.data };
} else {
throw {
message: 'Error Initializing :: please try again !!',
};
}
}
async function readTemplate(templateId) {
if (!info.initialized) {
await initialize('shrO8aYf3ybwSdDKn');
}
const resreq = await axios(
`https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
{
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
referrer: 'https://www.airtable.com/',
referrerPolicy: 'same-origin',
body: null,
method: 'GET',
mode: 'cors',
credentials: 'include',
}
)
.then((response) => {
return response.data;
})
.catch(() => {
throw {
message:
'Error Fetching :: Ensure www.airtable.com/templates/featured/<TemplateID> is accessible.',
};
});
return { template: resreq };
}
function unicodeToChar(text) {
return text.replace(/\\u[\dA-F]{4}/gi, function (match) {
return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16));
});
}
export default {
initialize,
read,
readView,
readTemplate,
};

2444
packages/nocodb-nest/src/modules/sync/helpers/job.ts

File diff suppressed because it is too large Load Diff

346
packages/nocodb-nest/src/modules/sync/helpers/readAndProcessData.ts

@ -0,0 +1,346 @@
import { RelationTypes, UITypes } from 'nocodb-sdk';
import { bulkDataService, tableService } from '../..';
import EntityMap from './EntityMap';
import type { AirtableBase } from 'airtable/lib/airtable_base';
import type { TableType } from 'nocodb-sdk';
const BULK_DATA_BATCH_SIZE = 500;
const ASSOC_BULK_DATA_BATCH_SIZE = 1000;
const BULK_PARALLEL_PROCESS = 5;
async function readAllData({
table,
fields,
base,
logBasic = (_str) => {},
}: {
table: { title?: string };
fields?;
base: AirtableBase;
logBasic?: (string) => void;
logDetailed?: (string) => void;
}): Promise<EntityMap> {
return new Promise((resolve, reject) => {
let data = null;
const selectParams: any = {
pageSize: 100,
};
if (fields) selectParams.fields = fields;
base(table.title)
.select(selectParams)
.eachPage(
async function page(records, fetchNextPage) {
if (!data) {
data = new EntityMap();
await data.init();
}
for await (const record of records) {
await data.addRow({ id: record.id, ...record.fields });
}
const tmpLength = await data.getCount();
logBasic(
`:: Reading '${table.title}' data :: ${Math.max(
1,
tmpLength - records.length,
)} - ${tmpLength}`,
);
// To fetch the next page of records, call `fetchNextPage`.
// If there are more records, `page` will get called again.
// If there are no more records, `done` will get called.
fetchNextPage();
},
async function done(err) {
if (err) {
console.error(err);
return reject(err);
}
resolve(data);
},
);
});
}
export async function importData({
projectName,
table,
base,
nocoBaseDataProcessing_v2,
sDB,
logDetailed = (_str) => {},
logBasic = (_str) => {},
}: {
projectName: string;
table: { title?: string; id?: string };
fields?;
base: AirtableBase;
logBasic: (string) => void;
logDetailed: (string) => void;
nocoBaseDataProcessing_v2;
sDB;
}): Promise<EntityMap> {
try {
// @ts-ignore
const records = await readAllData({
table,
base,
logDetailed,
logBasic,
});
await new Promise(async (resolve) => {
const readable = records.getStream();
const allRecordsCount = await records.getCount();
const promises = [];
let tempData = [];
let importedCount = 0;
let activeProcess = 0;
readable.on('data', async (record) => {
promises.push(
new Promise(async (resolve) => {
activeProcess++;
if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause();
const { id: rid, ...fields } = record;
const r = await nocoBaseDataProcessing_v2(sDB, table, {
id: rid,
fields,
});
tempData.push(r);
if (tempData.length >= BULK_DATA_BATCH_SIZE) {
let insertArray = tempData.splice(0, tempData.length);
await bulkDataService.bulkDataInsert({
projectName,
tableName: table.title,
body: insertArray,
cookie: {},
});
logBasic(
`:: Importing '${
table.title
}' data :: ${importedCount} - ${Math.min(
importedCount + BULK_DATA_BATCH_SIZE,
allRecordsCount,
)}`,
);
importedCount += insertArray.length;
insertArray = [];
}
activeProcess--;
if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume();
resolve(true);
}),
);
});
readable.on('end', async () => {
await Promise.all(promises);
if (tempData.length > 0) {
await bulkDataService.bulkDataInsert({
projectName,
tableName: table.title,
body: tempData,
cookie: {},
});
logBasic(
`:: Importing '${
table.title
}' data :: ${importedCount} - ${Math.min(
importedCount + BULK_DATA_BATCH_SIZE,
allRecordsCount,
)}`,
);
importedCount += tempData.length;
tempData = [];
}
resolve(true);
});
});
return records;
} catch (e) {
console.log(e);
return null;
}
}
export async function importLTARData({
table,
fields,
base,
projectName,
insertedAssocRef = {},
logDetailed = (_str) => {},
logBasic = (_str) => {},
records,
atNcAliasRef,
ncLinkMappingTable,
syncDB,
}: {
projectName: string;
table: { title?: string; id?: string };
fields;
base: AirtableBase;
logDetailed: (string) => void;
logBasic: (string) => void;
insertedAssocRef: { [assocTableId: string]: boolean };
records?: EntityMap;
atNcAliasRef: {
[ncTableId: string]: {
[ncTitle: string]: string;
};
};
ncLinkMappingTable: Record<string, Record<string, any>>[];
syncDB;
}) {
const assocTableMetas: Array<{
modelMeta: { id?: string; title?: string };
colMeta: { title?: string };
curCol: { title?: string };
refCol: { title?: string };
}> = [];
const allData =
records ||
(await readAllData({
table,
fields,
base,
logDetailed,
logBasic,
}));
const modelMeta: any = await tableService.getTableWithAccessibleViews({
tableId: table.id,
user: syncDB.user,
});
for (const colMeta of modelMeta.columns) {
// skip columns which are not LTAR and Many to many
if (
colMeta.uidt !== UITypes.LinkToAnotherRecord ||
colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY
) {
continue;
}
// skip if already inserted
if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue;
// self links: skip if the column under consideration is the add-on column NocoDB creates
if (ncLinkMappingTable.every((a) => a.nc.title !== colMeta.title)) continue;
// mark as inserted
insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true;
const assocModelMeta: TableType =
(await tableService.getTableWithAccessibleViews({
tableId: colMeta.colOptions.fk_mm_model_id,
user: syncDB.user,
})) as any;
// extract associative table and columns meta
assocTableMetas.push({
modelMeta: assocModelMeta,
colMeta,
curCol: assocModelMeta.columns.find(
(c) => c.id === colMeta.colOptions.fk_mm_child_column_id,
),
refCol: assocModelMeta.columns.find(
(c) => c.id === colMeta.colOptions.fk_mm_parent_column_id,
),
});
}
let nestedLinkCnt = 0;
// Iterate over all related M2M associative table
for await (const assocMeta of assocTableMetas) {
let assocTableData = [];
let importedCount = 0;
// extract insert data from records
await new Promise((resolve) => {
const promises = [];
const readable = allData.getStream();
let activeProcess = 0;
readable.on('data', async (record) => {
promises.push(
new Promise(async (resolve) => {
activeProcess++;
if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause();
const { id: _atId, ...rec } = record;
// todo: use actual alias instead of sanitized
assocTableData.push(
...(
rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || []
).map((id) => ({
[assocMeta.curCol.title]: record.id,
[assocMeta.refCol.title]: id,
})),
);
if (assocTableData.length >= ASSOC_BULK_DATA_BATCH_SIZE) {
let insertArray = assocTableData.splice(0, assocTableData.length);
logBasic(
`:: Importing '${
table.title
}' LTAR data :: ${importedCount} - ${Math.min(
importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
insertArray.length,
)}`,
);
await bulkDataService.bulkDataInsert({
projectName,
tableName: assocMeta.modelMeta.title,
body: insertArray,
cookie: {},
});
importedCount += insertArray.length;
insertArray = [];
}
activeProcess--;
if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume();
resolve(true);
}),
);
});
readable.on('end', async () => {
await Promise.all(promises);
if (assocTableData.length >= 0) {
logBasic(
`:: Importing '${
table.title
}' LTAR data :: ${importedCount} - ${Math.min(
importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
assocTableData.length,
)}`,
);
await bulkDataService.bulkDataInsert({
projectName,
tableName: assocMeta.modelMeta.title,
body: assocTableData,
cookie: {},
});
importedCount += assocTableData.length;
assocTableData = [];
}
resolve(true);
});
});
nestedLinkCnt += importedCount;
}
return nestedLinkCnt;
}

31
packages/nocodb-nest/src/modules/sync/helpers/syncMap.ts

@ -0,0 +1,31 @@
export const mapTbl = {};
// static mapping records between aTblId && ncId
export const addToMappingTbl = function addToMappingTbl(
aTblId,
ncId,
ncName,
parent?
) {
mapTbl[aTblId] = {
ncId: ncId,
ncParent: parent,
// name added to assist in quick debug
ncName: ncName,
};
};
// get NcID from airtable ID
export const getNcIdFromAtId = function getNcIdFromAtId(aId) {
return mapTbl[aId]?.ncId;
};
// get nc Parent from airtable ID
export const getNcParentFromAtId = function getNcParentFromAtId(aId) {
return mapTbl[aId]?.ncParent;
};
// get nc-title from airtable ID
export const getNcNameFromAtId = function getNcNameFromAtId(aId) {
return mapTbl[aId]?.ncName;
};

20
packages/nocodb-nest/src/modules/sync/sync.controller.spec.ts

@ -0,0 +1,20 @@
import { Test, TestingModule } from '@nestjs/testing';
import { SyncController } from './sync.controller';
import { SyncService } from './sync.service';
describe('SyncController', () => {
let controller: SyncController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [SyncController],
providers: [SyncService],
}).compile();
controller = module.get<SyncController>(SyncController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

67
packages/nocodb-nest/src/modules/sync/sync.controller.ts

@ -0,0 +1,67 @@
import {
Body,
Controller,
Delete,
Get,
Param,
Patch,
Post,
Req,
} from '@nestjs/common';
import { Acl } from '../../middlewares/extract-project-id/extract-project-id.middleware';
import { SyncService } from './sync.service';
@Controller('sync')
export class SyncController {
constructor(private readonly syncService: SyncService) {}
@Get([
'/api/v1/db/meta/projects/:projectId/syncs',
'/api/v1/db/meta/projects/:projectId/syncs/:baseId',
])
@Acl('syncSourceList')
async syncSourceList(
@Param('projectId') projectId: string,
@Param('baseId') baseId?: string,
) {
return await this.syncService.syncSourceList({
projectId,
baseId,
});
}
@Post([
'/api/v1/db/meta/projects/:projectId/syncs',
'/api/v1/db/meta/projects/:projectId/syncs/:baseId',
])
@Acl('syncSourceCreate')
async syncCreate(
@Param('projectId') projectId: string,
@Body() body: any,
@Req() req,
@Param('baseId') baseId?: string,
) {
return await this.syncService.syncCreate({
projectId: projectId,
baseId: baseId,
userId: (req as any).user.id,
syncPayload: body,
});
}
@Delete('/api/v1/db/meta/syncs/:syncId')
@Acl('syncSourceDelete')
async syncDelete(@Param('syncId') syncId: string) {
return await this.syncService.syncDelete({
syncId: syncId,
});
}
@Patch('/api/v1/db/meta/syncs/:syncId')
async syncUpdate(@Param('syncId') syncId: string, @Body() body: any) {
return await this.syncService.syncUpdate({
syncId: syncId,
syncPayload: body,
});
}
}

9
packages/nocodb-nest/src/modules/sync/sync.module.ts

@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { SyncService } from './sync.service';
import { SyncController } from './sync.controller';
@Module({
controllers: [SyncController],
providers: [SyncService]
})
export class SyncModule {}

18
packages/nocodb-nest/src/modules/sync/sync.service.spec.ts

@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { SyncService } from './sync.service';
describe('SyncService', () => {
let service: SyncService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [SyncService],
}).compile();
service = module.get<SyncService>(SyncService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

45
packages/nocodb-nest/src/modules/sync/sync.service.ts

@ -0,0 +1,45 @@
import { Injectable } from '@nestjs/common';
import { PagedResponseImpl } from '../../helpers/PagedResponse';
import { Project, SyncSource } from '../../models';
import { T } from 'nc-help';
@Injectable()
export class SyncService {
async syncSourceList(param: { projectId: string; baseId?: string }) {
return new PagedResponseImpl(
await SyncSource.list(param.projectId, param.baseId),
);
}
async syncCreate(param: {
projectId: string;
baseId?: string;
userId: string;
syncPayload: Partial<SyncSource>;
}) {
T.emit('evt', { evt_type: 'syncSource:created' });
const project = await Project.getWithInfo(param.projectId);
const sync = await SyncSource.insert({
...param.syncPayload,
fk_user_id: param.userId,
base_id: param.baseId ? param.baseId : project.bases[0].id,
project_id: param.projectId,
});
return sync;
}
async syncDelete(param: { syncId: string }) {
T.emit('evt', { evt_type: 'syncSource:deleted' });
return await SyncSource.delete(param.syncId);
}
async syncUpdate(param: {
syncId: string;
syncPayload: Partial<SyncSource>;
}) {
T.emit('evt', { evt_type: 'syncSource:updated' });
return await SyncSource.update(param.syncId, param.syncPayload);
}
}

20
packages/nocodb-nest/src/modules/test/test.controller.spec.ts

@ -0,0 +1,20 @@
import { Test, TestingModule } from '@nestjs/testing';
import { TestController } from './test.controller';
import { TestService } from './test.service';
describe('TestController', () => {
let controller: TestController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [TestController],
providers: [TestService],
}).compile();
controller = module.get<TestController>(TestController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

7
packages/nocodb-nest/src/modules/test/test.controller.ts

@ -0,0 +1,7 @@
import { Controller } from '@nestjs/common';
import { TestService } from './test.service';
@Controller('test')
export class TestController {
constructor(private readonly testService: TestService) {}
}

9
packages/nocodb-nest/src/modules/test/test.module.ts

@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { TestService } from './test.service';
import { TestController } from './test.controller';
@Module({
controllers: [TestController],
providers: [TestService]
})
export class TestModule {}

18
packages/nocodb-nest/src/modules/test/test.service.spec.ts

@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { TestService } from './test.service';
describe('TestService', () => {
let service: TestService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [TestService],
}).compile();
service = module.get<TestService>(TestService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

4
packages/nocodb-nest/src/modules/test/test.service.ts

@ -0,0 +1,4 @@
import { Injectable } from '@nestjs/common';
@Injectable()
export class TestService {}
Loading…
Cancel
Save