mirror of https://github.com/nocodb/nocodb
mertmit
2 years ago
21 changed files with 2708 additions and 2775 deletions
@ -1,6 +0,0 @@
|
||||
export abstract class NocoSyncSourceAdapter { |
||||
public abstract init(): Promise<void>; |
||||
public abstract destProjectWrite(): Promise<any>; |
||||
public abstract destSchemaWrite(): Promise<any>; |
||||
public abstract destDataWrite(): Promise<any>; |
||||
} |
||||
@ -1,7 +0,0 @@
|
||||
export abstract class NocoSyncSourceAdapter { |
||||
public abstract init(): Promise<void>; |
||||
public abstract srcSchemaGet(): Promise<any>; |
||||
public abstract srcDataLoad(): Promise<any>; |
||||
public abstract srcDataListen(): Promise<any>; |
||||
public abstract srcDataPoll(): Promise<any>; |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -1,21 +0,0 @@
|
||||
import { Test } from '@nestjs/testing'; |
||||
import { ImportService } from '../../services/import.service'; |
||||
import { ImportController } from './import.controller'; |
||||
import type { TestingModule } from '@nestjs/testing'; |
||||
|
||||
describe('ImportController', () => { |
||||
let controller: ImportController; |
||||
|
||||
beforeEach(async () => { |
||||
const module: TestingModule = await Test.createTestingModule({ |
||||
controllers: [ImportController], |
||||
providers: [ImportService], |
||||
}).compile(); |
||||
|
||||
controller = module.get<ImportController>(ImportController); |
||||
}); |
||||
|
||||
it('should be defined', () => { |
||||
expect(controller).toBeDefined(); |
||||
}); |
||||
}); |
||||
@ -1,148 +0,0 @@
|
||||
import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common'; |
||||
import { forwardRef, Inject } from '@nestjs/common'; |
||||
import { ModuleRef } from '@nestjs/core'; |
||||
import { SocketGateway } from 'src/services/socket.gateway'; |
||||
import { GlobalGuard } from '../../guards/global/global.guard'; |
||||
import { NcError } from '../../helpers/catchError'; |
||||
import { ExtractProjectIdMiddleware } from '../../middlewares/extract-project-id/extract-project-id.middleware'; |
||||
import { SyncSource } from '../../models'; |
||||
import NocoJobs from '../../jobs/NocoJobs'; |
||||
import airtableSyncJob from './helpers/job'; |
||||
import type { AirtableSyncConfig } from './helpers/job'; |
||||
|
||||
import type { Server } from 'socket.io'; |
||||
|
||||
const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB'; |
||||
const AIRTABLE_PROGRESS_JOB = 'AIRTABLE_PROGRESS_JOB'; |
||||
|
||||
enum SyncStatus { |
||||
PROGRESS = 'PROGRESS', |
||||
COMPLETED = 'COMPLETED', |
||||
FAILED = 'FAILED', |
||||
} |
||||
|
||||
const initJob = (sv: Server, jobs: { [p: string]: { last_message: any } }) => { |
||||
// add importer job handler and progress notification job handler
|
||||
NocoJobs.jobsMgr.addJobWorker(AIRTABLE_IMPORT_JOB, airtableSyncJob); |
||||
NocoJobs.jobsMgr.addJobWorker( |
||||
AIRTABLE_PROGRESS_JOB, |
||||
({ payload, progress }) => { |
||||
sv.to(payload?.id).emit('progress', { |
||||
msg: progress?.msg, |
||||
level: progress?.level, |
||||
status: progress?.status, |
||||
}); |
||||
|
||||
if (payload?.id in jobs) { |
||||
jobs[payload?.id].last_message = { |
||||
msg: progress?.msg, |
||||
level: progress?.level, |
||||
status: progress?.status, |
||||
}; |
||||
} |
||||
}, |
||||
); |
||||
|
||||
NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => { |
||||
NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { |
||||
payload, |
||||
progress: { |
||||
msg: progress?.msg, |
||||
level: progress?.level, |
||||
status: progress?.status, |
||||
}, |
||||
}); |
||||
}); |
||||
NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, (payload) => { |
||||
NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { |
||||
payload, |
||||
progress: { |
||||
msg: 'Complete!', |
||||
status: SyncStatus.COMPLETED, |
||||
}, |
||||
}); |
||||
delete jobs[payload?.id]; |
||||
}); |
||||
NocoJobs.jobsMgr.addFailureCbk(AIRTABLE_IMPORT_JOB, (payload, error: any) => { |
||||
NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { |
||||
payload, |
||||
progress: { |
||||
msg: error?.message || 'Failed due to some internal error', |
||||
status: SyncStatus.FAILED, |
||||
}, |
||||
}); |
||||
delete jobs[payload?.id]; |
||||
}); |
||||
}; |
||||
@Controller() |
||||
@UseGuards(ExtractProjectIdMiddleware, GlobalGuard) |
||||
export class ImportController { |
||||
constructor( |
||||
private readonly socketGateway: SocketGateway, |
||||
@Inject(forwardRef(() => ModuleRef)) private readonly moduleRef: ModuleRef, |
||||
) {} |
||||
|
||||
@Post('/api/v1/db/meta/import/airtable') |
||||
@HttpCode(200) |
||||
importAirtable(@Request() req) { |
||||
NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { |
||||
id: req.query.id, |
||||
...req.body, |
||||
}); |
||||
return {}; |
||||
} |
||||
|
||||
@Post('/api/v1/db/meta/syncs/:syncId/trigger') |
||||
@HttpCode(200) |
||||
async triggerSync(@Request() req) { |
||||
if (req.params.syncId in this.socketGateway.jobs) { |
||||
NcError.badRequest('Sync already in progress'); |
||||
} |
||||
|
||||
const syncSource = await SyncSource.get(req.params.syncId); |
||||
|
||||
const user = await syncSource.getUser(); |
||||
|
||||
// Treat default baseUrl as siteUrl from req object
|
||||
let baseURL = (req as any).ncSiteUrl; |
||||
|
||||
// if environment value avail use it
|
||||
// or if it's docker construct using `PORT`
|
||||
if (process.env.NC_DOCKER) { |
||||
baseURL = `http://localhost:${process.env.PORT || 8080}`; |
||||
} |
||||
|
||||
setTimeout(() => { |
||||
NocoJobs.jobsMgr.add<AirtableSyncConfig>(AIRTABLE_IMPORT_JOB, { |
||||
id: req.params.syncId, |
||||
...(syncSource?.details || {}), |
||||
projectId: syncSource.project_id, |
||||
baseId: syncSource.base_id, |
||||
authToken: '', |
||||
baseURL, |
||||
user: user, |
||||
moduleRef: this.moduleRef, |
||||
}); |
||||
}, 1000); |
||||
|
||||
this.socketGateway.jobs[req.params.syncId] = { |
||||
last_message: { |
||||
msg: 'Sync started', |
||||
}, |
||||
}; |
||||
return {}; |
||||
} |
||||
|
||||
@Post('/api/v1/db/meta/syncs/:syncId/abort') |
||||
@HttpCode(200) |
||||
async abortImport(@Request() req) { |
||||
if (req.params.syncId in this.socketGateway.jobs) { |
||||
delete this.socketGateway.jobs[req.params.syncId]; |
||||
} |
||||
return {}; |
||||
} |
||||
|
||||
async onModuleInit() { |
||||
initJob(this.socketGateway.io, this.socketGateway.jobs); |
||||
} |
||||
} |
||||
@ -0,0 +1,76 @@
|
||||
import { InjectQueue } from '@nestjs/bull'; |
||||
import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common'; |
||||
import { GlobalGuard } from 'src/guards/global/global.guard'; |
||||
import { ExtractProjectIdMiddleware } from 'src/middlewares/extract-project-id/extract-project-id.middleware'; |
||||
import { Queue } from 'bull'; |
||||
import { SyncSource } from 'src/models'; |
||||
import { NcError } from 'src/helpers/catchError'; |
||||
import { QueueService } from '../fallback-queue.service'; |
||||
import { JobsService } from '../jobs.service'; |
||||
|
||||
@Controller() |
||||
@UseGuards(ExtractProjectIdMiddleware, GlobalGuard) |
||||
export class AtImportController { |
||||
activeQueue; |
||||
constructor( |
||||
@InjectQueue('jobs') private readonly jobsQueue: Queue, |
||||
private readonly fallbackQueueService: QueueService, |
||||
private readonly jobsService: JobsService, |
||||
) { |
||||
this.activeQueue = process.env.NC_REDIS_URL |
||||
? this.jobsQueue |
||||
: this.fallbackQueueService; |
||||
} |
||||
|
||||
@Post('/api/v1/db/meta/import/airtable') |
||||
@HttpCode(200) |
||||
async importAirtable(@Request() req) { |
||||
const job = await this.activeQueue.add('at-import', { |
||||
...req.body, |
||||
}); |
||||
|
||||
return { id: job.id, name: job.name }; |
||||
} |
||||
|
||||
@Post('/api/v1/db/meta/syncs/:syncId/trigger') |
||||
@HttpCode(200) |
||||
async triggerSync(@Request() req) { |
||||
const jobs = await this.jobsService.jobList('at-import'); |
||||
const fnd = jobs.find((j) => j.data.syncId === req.params.syncId); |
||||
|
||||
if (fnd) { |
||||
NcError.badRequest('Sync already in progress'); |
||||
} |
||||
|
||||
const syncSource = await SyncSource.get(req.params.syncId); |
||||
|
||||
const user = await syncSource.getUser(); |
||||
|
||||
// Treat default baseUrl as siteUrl from req object
|
||||
let baseURL = (req as any).ncSiteUrl; |
||||
|
||||
// if environment value avail use it
|
||||
// or if it's docker construct using `PORT`
|
||||
if (process.env.NC_DOCKER) { |
||||
baseURL = `http://localhost:${process.env.PORT || 8080}`; |
||||
} |
||||
|
||||
const job = await this.activeQueue.add('at-import', { |
||||
syncId: req.params.syncId, |
||||
...(syncSource?.details || {}), |
||||
projectId: syncSource.project_id, |
||||
baseId: syncSource.base_id, |
||||
authToken: '', |
||||
baseURL, |
||||
user: user, |
||||
}); |
||||
|
||||
return { id: job.id, name: job.name }; |
||||
} |
||||
|
||||
@Post('/api/v1/db/meta/syncs/:syncId/abort') |
||||
@HttpCode(200) |
||||
async abortImport(@Request() req) { |
||||
return {}; |
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -1,7 +1,8 @@
|
||||
/* eslint-disable no-async-promise-executor */ |
||||
import { RelationTypes, UITypes } from 'nocodb-sdk'; |
||||
import EntityMap from './EntityMap'; |
||||
import type { BulkDataAliasService } from '../../../services/bulk-data-alias.service'; |
||||
import type { TablesService } from '../../../services/tables.service'; |
||||
import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service'; |
||||
import type { TablesService } from '../../../../services/tables.service'; |
||||
// @ts-ignore
|
||||
import type { AirtableBase } from 'airtable/lib/airtable_base'; |
||||
import type { TableType } from 'nocodb-sdk'; |
||||
Loading…
Reference in new issue