mirror of https://github.com/nocodb/nocodb
starbirdtech383
2 years ago
11 changed files with 19783 additions and 37688 deletions
File diff suppressed because it is too large
Load Diff
@ -1,21 +0,0 @@ |
|||||||
import { Test } from '@nestjs/testing'; |
|
||||||
import { ExportImportService } from './../services/export-import.service'; |
|
||||||
import { ExportImportController } from './export-import.controller'; |
|
||||||
import type { TestingModule } from '@nestjs/testing'; |
|
||||||
|
|
||||||
describe('ExportImportController', () => { |
|
||||||
let controller: ExportImportController; |
|
||||||
|
|
||||||
beforeEach(async () => { |
|
||||||
const module: TestingModule = await Test.createTestingModule({ |
|
||||||
controllers: [ExportImportController], |
|
||||||
providers: [ExportImportService], |
|
||||||
}).compile(); |
|
||||||
|
|
||||||
controller = module.get<ExportImportController>(ExportImportController); |
|
||||||
}); |
|
||||||
|
|
||||||
it('should be defined', () => { |
|
||||||
expect(controller).toBeDefined(); |
|
||||||
}); |
|
||||||
}); |
|
||||||
@ -1,51 +0,0 @@ |
|||||||
import { |
|
||||||
Body, |
|
||||||
Controller, |
|
||||||
HttpCode, |
|
||||||
Param, |
|
||||||
Patch, |
|
||||||
Post, |
|
||||||
Request, |
|
||||||
UseGuards, |
|
||||||
} from '@nestjs/common'; |
|
||||||
import { AuthGuard } from '@nestjs/passport'; |
|
||||||
import { GlobalGuard } from '../guards/global/global.guard'; |
|
||||||
import { |
|
||||||
Acl, |
|
||||||
ExtractProjectIdMiddleware, |
|
||||||
} from '../middlewares/extract-project-id/extract-project-id.middleware'; |
|
||||||
import { ExportImportService } from './../services/export-import.service'; |
|
||||||
|
|
||||||
@Controller() |
|
||||||
@UseGuards(ExtractProjectIdMiddleware, GlobalGuard) |
|
||||||
export class ExportImportController { |
|
||||||
constructor(private readonly exportImportService: ExportImportService) {} |
|
||||||
|
|
||||||
@Post('/api/v1/db/meta/export/:projectId/:baseId') |
|
||||||
@HttpCode(200) |
|
||||||
@Acl('exportBase') |
|
||||||
async exportBase(@Param('baseId') baseId: string, @Body() body: any) { |
|
||||||
return await this.exportImportService.exportBase({ |
|
||||||
baseId, |
|
||||||
path: body.path, |
|
||||||
}); |
|
||||||
} |
|
||||||
|
|
||||||
@Post('/api/v1/db/meta/import/:projectId/:baseId') |
|
||||||
@HttpCode(200) |
|
||||||
@Acl('importBase') |
|
||||||
async importBase( |
|
||||||
@Request() req, |
|
||||||
@Param('projectId') projectId: string, |
|
||||||
@Param('baseId') baseId: string, |
|
||||||
@Body() body: any, |
|
||||||
) { |
|
||||||
return await this.exportImportService.importBase({ |
|
||||||
user: (req as any).user, |
|
||||||
projectId: projectId, |
|
||||||
baseId: baseId, |
|
||||||
src: body.src, |
|
||||||
req, |
|
||||||
}); |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,6 +0,0 @@ |
|||||||
export abstract class NocoSyncSourceAdapter { |
|
||||||
public abstract init(): Promise<void>; |
|
||||||
public abstract destProjectWrite(): Promise<any>; |
|
||||||
public abstract destSchemaWrite(): Promise<any>; |
|
||||||
public abstract destDataWrite(): Promise<any>; |
|
||||||
} |
|
||||||
@ -1,7 +0,0 @@ |
|||||||
export abstract class NocoSyncSourceAdapter { |
|
||||||
public abstract init(): Promise<void>; |
|
||||||
public abstract srcSchemaGet(): Promise<any>; |
|
||||||
public abstract srcDataLoad(): Promise<any>; |
|
||||||
public abstract srcDataListen(): Promise<any>; |
|
||||||
public abstract srcDataPoll(): Promise<any>; |
|
||||||
} |
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1,21 +0,0 @@ |
|||||||
import { Test } from '@nestjs/testing'; |
|
||||||
import { ImportService } from '../../services/import.service'; |
|
||||||
import { ImportController } from './import.controller'; |
|
||||||
import type { TestingModule } from '@nestjs/testing'; |
|
||||||
|
|
||||||
describe('ImportController', () => { |
|
||||||
let controller: ImportController; |
|
||||||
|
|
||||||
beforeEach(async () => { |
|
||||||
const module: TestingModule = await Test.createTestingModule({ |
|
||||||
controllers: [ImportController], |
|
||||||
providers: [ImportService], |
|
||||||
}).compile(); |
|
||||||
|
|
||||||
controller = module.get<ImportController>(ImportController); |
|
||||||
}); |
|
||||||
|
|
||||||
it('should be defined', () => { |
|
||||||
expect(controller).toBeDefined(); |
|
||||||
}); |
|
||||||
}); |
|
||||||
@ -1,148 +0,0 @@ |
|||||||
import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common'; |
|
||||||
import { forwardRef, Inject } from '@nestjs/common'; |
|
||||||
import { ModuleRef } from '@nestjs/core'; |
|
||||||
import { SocketGateway } from 'src/services/socket.gateway'; |
|
||||||
import { GlobalGuard } from '../../guards/global/global.guard'; |
|
||||||
import { NcError } from '../../helpers/catchError'; |
|
||||||
import { ExtractProjectIdMiddleware } from '../../middlewares/extract-project-id/extract-project-id.middleware'; |
|
||||||
import { SyncSource } from '../../models'; |
|
||||||
import NocoJobs from '../../jobs/NocoJobs'; |
|
||||||
import airtableSyncJob from './helpers/job'; |
|
||||||
import type { AirtableSyncConfig } from './helpers/job'; |
|
||||||
|
|
||||||
import type { Server } from 'socket.io'; |
|
||||||
|
|
||||||
const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB'; |
|
||||||
const AIRTABLE_PROGRESS_JOB = 'AIRTABLE_PROGRESS_JOB'; |
|
||||||
|
|
||||||
enum SyncStatus { |
|
||||||
PROGRESS = 'PROGRESS', |
|
||||||
COMPLETED = 'COMPLETED', |
|
||||||
FAILED = 'FAILED', |
|
||||||
} |
|
||||||
|
|
||||||
const initJob = (sv: Server, jobs: { [p: string]: { last_message: any } }) => { |
|
||||||
// add importer job handler and progress notification job handler
|
|
||||||
NocoJobs.jobsMgr.addJobWorker(AIRTABLE_IMPORT_JOB, airtableSyncJob); |
|
||||||
NocoJobs.jobsMgr.addJobWorker( |
|
||||||
AIRTABLE_PROGRESS_JOB, |
|
||||||
({ payload, progress }) => { |
|
||||||
sv.to(payload?.id).emit('progress', { |
|
||||||
msg: progress?.msg, |
|
||||||
level: progress?.level, |
|
||||||
status: progress?.status, |
|
||||||
}); |
|
||||||
|
|
||||||
if (payload?.id in jobs) { |
|
||||||
jobs[payload?.id].last_message = { |
|
||||||
msg: progress?.msg, |
|
||||||
level: progress?.level, |
|
||||||
status: progress?.status, |
|
||||||
}; |
|
||||||
} |
|
||||||
}, |
|
||||||
); |
|
||||||
|
|
||||||
NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => { |
|
||||||
NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { |
|
||||||
payload, |
|
||||||
progress: { |
|
||||||
msg: progress?.msg, |
|
||||||
level: progress?.level, |
|
||||||
status: progress?.status, |
|
||||||
}, |
|
||||||
}); |
|
||||||
}); |
|
||||||
NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, (payload) => { |
|
||||||
NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { |
|
||||||
payload, |
|
||||||
progress: { |
|
||||||
msg: 'Complete!', |
|
||||||
status: SyncStatus.COMPLETED, |
|
||||||
}, |
|
||||||
}); |
|
||||||
delete jobs[payload?.id]; |
|
||||||
}); |
|
||||||
NocoJobs.jobsMgr.addFailureCbk(AIRTABLE_IMPORT_JOB, (payload, error: any) => { |
|
||||||
NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { |
|
||||||
payload, |
|
||||||
progress: { |
|
||||||
msg: error?.message || 'Failed due to some internal error', |
|
||||||
status: SyncStatus.FAILED, |
|
||||||
}, |
|
||||||
}); |
|
||||||
delete jobs[payload?.id]; |
|
||||||
}); |
|
||||||
}; |
|
||||||
@Controller() |
|
||||||
@UseGuards(ExtractProjectIdMiddleware, GlobalGuard) |
|
||||||
export class ImportController { |
|
||||||
constructor( |
|
||||||
private readonly socketGateway: SocketGateway, |
|
||||||
@Inject(forwardRef(() => ModuleRef)) private readonly moduleRef: ModuleRef, |
|
||||||
) {} |
|
||||||
|
|
||||||
@Post('/api/v1/db/meta/import/airtable') |
|
||||||
@HttpCode(200) |
|
||||||
importAirtable(@Request() req) { |
|
||||||
NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { |
|
||||||
id: req.query.id, |
|
||||||
...req.body, |
|
||||||
}); |
|
||||||
return {}; |
|
||||||
} |
|
||||||
|
|
||||||
@Post('/api/v1/db/meta/syncs/:syncId/trigger') |
|
||||||
@HttpCode(200) |
|
||||||
async triggerSync(@Request() req) { |
|
||||||
if (req.params.syncId in this.socketGateway.jobs) { |
|
||||||
NcError.badRequest('Sync already in progress'); |
|
||||||
} |
|
||||||
|
|
||||||
const syncSource = await SyncSource.get(req.params.syncId); |
|
||||||
|
|
||||||
const user = await syncSource.getUser(); |
|
||||||
|
|
||||||
// Treat default baseUrl as siteUrl from req object
|
|
||||||
let baseURL = (req as any).ncSiteUrl; |
|
||||||
|
|
||||||
// if environment value avail use it
|
|
||||||
// or if it's docker construct using `PORT`
|
|
||||||
if (process.env.NC_DOCKER) { |
|
||||||
baseURL = `http://localhost:${process.env.PORT || 8080}`; |
|
||||||
} |
|
||||||
|
|
||||||
setTimeout(() => { |
|
||||||
NocoJobs.jobsMgr.add<AirtableSyncConfig>(AIRTABLE_IMPORT_JOB, { |
|
||||||
id: req.params.syncId, |
|
||||||
...(syncSource?.details || {}), |
|
||||||
projectId: syncSource.project_id, |
|
||||||
baseId: syncSource.base_id, |
|
||||||
authToken: '', |
|
||||||
baseURL, |
|
||||||
user: user, |
|
||||||
moduleRef: this.moduleRef, |
|
||||||
}); |
|
||||||
}, 1000); |
|
||||||
|
|
||||||
this.socketGateway.jobs[req.params.syncId] = { |
|
||||||
last_message: { |
|
||||||
msg: 'Sync started', |
|
||||||
}, |
|
||||||
}; |
|
||||||
return {}; |
|
||||||
} |
|
||||||
|
|
||||||
@Post('/api/v1/db/meta/syncs/:syncId/abort') |
|
||||||
@HttpCode(200) |
|
||||||
async abortImport(@Request() req) { |
|
||||||
if (req.params.syncId in this.socketGateway.jobs) { |
|
||||||
delete this.socketGateway.jobs[req.params.syncId]; |
|
||||||
} |
|
||||||
return {}; |
|
||||||
} |
|
||||||
|
|
||||||
async onModuleInit() { |
|
||||||
initJob(this.socketGateway.io, this.socketGateway.jobs); |
|
||||||
} |
|
||||||
} |
|
||||||
Loading…
Reference in new issue