mirror of https://github.com/nocodb/nocodb
Browse Source
* feat: csv export job * fix: duplicate job * feat: csv export job final * feat: data export extension POC * feat: data export final * fix: extensions & scroll --------- Co-authored-by: Raju Udava <86527202+dstala@users.noreply.github.com>pull/8891/head
Mert E
5 months ago
committed by
GitHub
46 changed files with 1455 additions and 315 deletions
@ -0,0 +1,63 @@
|
||||
const jobsState = createGlobalState(() => { |
||||
const baseJobs = ref<Record<string, JobType[]>>({}) |
||||
return { baseJobs } |
||||
}) |
||||
|
||||
interface JobType { |
||||
id: string |
||||
job: string |
||||
status: string |
||||
result: Record<string, any> |
||||
fk_user_id: string |
||||
fk_workspace_id: string |
||||
base_id: string |
||||
created_at: Date |
||||
updated_at: Date |
||||
} |
||||
|
||||
export const useJobs = createSharedComposable(() => { |
||||
const { baseJobs } = jobsState() |
||||
|
||||
const { $api } = useNuxtApp() |
||||
|
||||
const { base } = storeToRefs(useBase()) |
||||
|
||||
const activeBaseJobs = computed(() => { |
||||
if (!base.value || !base.value.id) { |
||||
return null |
||||
} |
||||
return baseJobs.value[base.value.id] |
||||
}) |
||||
|
||||
const jobList = computed<JobType[]>(() => { |
||||
return activeBaseJobs.value || [] |
||||
}) |
||||
|
||||
const getJobsForBase = (baseId: string) => { |
||||
return baseJobs.value[baseId] || [] |
||||
} |
||||
|
||||
const loadJobsForBase = async (baseId?: string) => { |
||||
if (!baseId) { |
||||
baseId = base.value.id |
||||
|
||||
if (!baseId) { |
||||
return |
||||
} |
||||
} |
||||
|
||||
const jobs: JobType[] = await $api.jobs.list(baseId, {}) |
||||
|
||||
if (baseJobs.value[baseId]) { |
||||
baseJobs.value[baseId] = jobs || baseJobs.value[baseId] |
||||
} else { |
||||
baseJobs.value[baseId] = jobs || [] |
||||
} |
||||
} |
||||
|
||||
return { |
||||
jobList, |
||||
loadJobsForBase, |
||||
getJobsForBase, |
||||
} |
||||
}) |
||||
|
After Width: | Height: | Size: 36 KiB |
@ -0,0 +1,180 @@
|
||||
<script setup lang="ts"> |
||||
import dayjs from 'dayjs' |
||||
import { type ViewType, ViewTypes } from 'nocodb-sdk' |
||||
|
||||
const { $api, $poller } = useNuxtApp() |
||||
|
||||
const { appInfo } = useGlobal() |
||||
|
||||
const { extension, tables, fullscreen, getViewsForTable } = useExtensionHelperOrThrow() |
||||
|
||||
const { jobList, loadJobsForBase } = useJobs() |
||||
|
||||
const views = ref<ViewType[]>([]) |
||||
|
||||
const exportedFiles = computed(() => { |
||||
return jobList.value |
||||
.filter((job) => job.job === 'data-export') |
||||
.map((job) => { |
||||
return { |
||||
...job, |
||||
result: (job.result || {}) as { url: string; type: 'csv' | 'json' | 'xlsx'; title: string; timestamp: number }, |
||||
} |
||||
}) |
||||
.sort((a, b) => dayjs(b.created_at).unix() - dayjs(a.created_at).unix()) |
||||
}) |
||||
|
||||
const exportPayload = ref<{ |
||||
tableId?: string |
||||
viewId?: string |
||||
}>({}) |
||||
|
||||
const tableList = computed(() => { |
||||
return tables.value.map((table) => { |
||||
return { |
||||
label: table.title, |
||||
value: table.id, |
||||
} |
||||
}) |
||||
}) |
||||
|
||||
const viewList = computed(() => { |
||||
if (!exportPayload.value.tableId) return [] |
||||
return ( |
||||
views.value |
||||
.filter((view) => view.type === ViewTypes.GRID) |
||||
.map((view) => { |
||||
return { |
||||
label: view.is_default ? `Default View` : view.title, |
||||
value: view.id, |
||||
} |
||||
}) || [] |
||||
) |
||||
}) |
||||
|
||||
const reloadViews = async () => { |
||||
if (exportPayload.value.tableId) { |
||||
views.value = await getViewsForTable(exportPayload.value.tableId) |
||||
} |
||||
} |
||||
|
||||
const onTableSelect = async (tableId: string) => { |
||||
exportPayload.value.tableId = tableId |
||||
await reloadViews() |
||||
exportPayload.value.viewId = views.value.find((view) => view.is_default)?.id |
||||
await extension.value.kvStore.set('exportPayload', exportPayload.value) |
||||
} |
||||
|
||||
const onViewSelect = async (viewId: string) => { |
||||
exportPayload.value.viewId = viewId |
||||
await extension.value.kvStore.set('exportPayload', exportPayload.value) |
||||
} |
||||
|
||||
const isExporting = ref(false) |
||||
|
||||
async function exportDataAsync() { |
||||
try { |
||||
if (isExporting.value || !exportPayload.value.viewId) return |
||||
|
||||
isExporting.value = true |
||||
|
||||
const jobData = await $api.export.data(exportPayload.value.viewId, 'csv', {}) |
||||
|
||||
jobList.value.unshift(jobData) |
||||
|
||||
$poller.subscribe( |
||||
{ id: jobData.id }, |
||||
async (data: { |
||||
id: string |
||||
status?: string |
||||
data?: { |
||||
error?: { |
||||
message: string |
||||
} |
||||
message?: string |
||||
result?: any |
||||
} |
||||
}) => { |
||||
if (data.status !== 'close') { |
||||
if (data.status === JobStatus.COMPLETED) { |
||||
// Export completed successfully |
||||
message.info('Successfully exported data!') |
||||
|
||||
const job = jobList.value.find((j) => j.id === data.id) |
||||
if (job) { |
||||
job.status = JobStatus.COMPLETED |
||||
job.result = data.data?.result |
||||
} |
||||
|
||||
isExporting.value = false |
||||
} else if (data.status === JobStatus.FAILED) { |
||||
message.error('Failed to export data!') |
||||
|
||||
const job = jobList.value.find((j) => j.id === data.id) |
||||
if (job) { |
||||
job.status = JobStatus.FAILED |
||||
} |
||||
|
||||
isExporting.value = false |
||||
} |
||||
} |
||||
}, |
||||
) |
||||
} catch (e: any) { |
||||
message.error(await extractSdkResponseErrorMsg(e)) |
||||
} |
||||
} |
||||
|
||||
const urlHelper = (url: string) => { |
||||
if (url.startsWith('http')) { |
||||
return url |
||||
} else { |
||||
return `${appInfo.value.ncSiteUrl || BASE_FALLBACK_URL}/${url}` |
||||
} |
||||
} |
||||
|
||||
const titleHelper = () => { |
||||
const table = tables.value.find((t) => t.id === exportPayload.value.tableId) |
||||
const view = views.value.find((v) => v.id === exportPayload.value.viewId) |
||||
|
||||
return `${table?.title} (${view?.is_default ? 'Default View' : view?.title})` |
||||
} |
||||
|
||||
onMounted(() => { |
||||
exportPayload.value = extension.value.kvStore.get('exportPayload') || {} |
||||
reloadViews() |
||||
loadJobsForBase() |
||||
}) |
||||
</script> |
||||
|
||||
<template> |
||||
<div class="flex flex-col gap-2 p-2"> |
||||
<NcSelect v-model:value="exportPayload.tableId" :options="tableList" @disabled="isExporting" @change="onTableSelect" /> |
||||
<NcSelect v-model:value="exportPayload.viewId" :options="viewList" @disabled="isExporting" @change="onViewSelect" /> |
||||
<NcButton @loading="isExporting" @click="exportDataAsync">Export</NcButton> |
||||
<div |
||||
class="flex flex-col" |
||||
:class="{ |
||||
'max-h-[60px] overflow-auto': !fullscreen, |
||||
}" |
||||
> |
||||
<div v-for="exp in exportedFiles" :key="exp.id" class="flex items-center gap-1"> |
||||
<template v-if="exp.status === JobStatus.COMPLETED && exp.result"> |
||||
<GeneralIcon icon="file" /> |
||||
<div>{{ exp.result.title }}</div> |
||||
<a :href="urlHelper(exp.result.url)" target="_blank">Download</a> |
||||
</template> |
||||
<template v-else-if="exp.status === JobStatus.FAILED"> |
||||
<GeneralIcon icon="error" class="text-red-500" /> |
||||
<div>{{ exp.result.title }}</div> |
||||
</template> |
||||
<template v-else> |
||||
<GeneralLoader size="small" /> |
||||
<div>{{ titleHelper() }}</div> |
||||
</template> |
||||
</div> |
||||
</div> |
||||
</div> |
||||
</template> |
||||
|
||||
<style lang="scss"></style> |
||||
@ -0,0 +1,11 @@
|
||||
{ |
||||
"id": "nc-data-exporter", |
||||
"title": "Data Exporter", |
||||
"description": "Export any view in various formats", |
||||
"entry": "data-exporter", |
||||
"version": "0.1", |
||||
"iconUrl": "data-exporter/icon.png", |
||||
"publisherName": "NocoDB", |
||||
"publisherEmail": "contact@nocodb.com", |
||||
"publisherUrl": "https://www.nocodb.com" |
||||
} |
||||
@ -0,0 +1,21 @@
|
||||
import { Test } from '@nestjs/testing'; |
||||
import { HooksService } from '../services/hooks.service'; |
||||
import { JobsMetaController } from './jobs-meta.controller'; |
||||
import type { TestingModule } from '@nestjs/testing'; |
||||
|
||||
describe('JobsMetaController', () => { |
||||
let controller: JobsMetaController; |
||||
|
||||
beforeEach(async () => { |
||||
const module: TestingModule = await Test.createTestingModule({ |
||||
controllers: [JobsMetaController], |
||||
providers: [HooksService], |
||||
}).compile(); |
||||
|
||||
controller = module.get<JobsMetaController>(JobsMetaController); |
||||
}); |
||||
|
||||
it('should be defined', () => { |
||||
expect(controller).toBeDefined(); |
||||
}); |
||||
}); |
||||
@ -0,0 +1,28 @@
|
||||
import { Body, Controller, Post, Req, UseGuards } from '@nestjs/common'; |
||||
import type { JobStatus, JobTypes } from '~/interface/Jobs'; |
||||
import { GlobalGuard } from '~/guards/global/global.guard'; |
||||
import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware'; |
||||
import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard'; |
||||
import { TenantContext } from '~/decorators/tenant-context.decorator'; |
||||
import { NcContext, NcRequest } from '~/interface/config'; |
||||
import { JobsMetaService } from '~/services/jobs-meta.service'; |
||||
|
||||
@Controller() |
||||
@UseGuards(MetaApiLimiterGuard, GlobalGuard) |
||||
export class JobsMetaController { |
||||
constructor(private readonly jobsMetaService: JobsMetaService) {} |
||||
|
||||
@Post(['/api/v2/jobs/:baseId']) |
||||
@Acl('jobList') |
||||
async jobList( |
||||
@TenantContext() context: NcContext, |
||||
@Req() req: NcRequest, |
||||
@Body() |
||||
conditions?: { |
||||
job?: JobTypes; |
||||
status?: JobStatus; |
||||
}, |
||||
) { |
||||
return await this.jobsMetaService.list(context, conditions, req); |
||||
} |
||||
} |
||||
@ -0,0 +1,30 @@
|
||||
import type { Knex } from 'knex'; |
||||
import { MetaTable } from '~/utils/globals'; |
||||
|
||||
const up = async (knex: Knex) => { |
||||
await knex.schema.createTable(MetaTable.JOBS, (table) => { |
||||
table.string('id', 20).primary(); |
||||
|
||||
table.string('job', 255); |
||||
|
||||
table.string('status', 20); |
||||
|
||||
table.text('result'); |
||||
|
||||
table.string('fk_user_id', 20); |
||||
|
||||
table.string('fk_workspace_id', 20); |
||||
|
||||
table.string('base_id', 20); |
||||
|
||||
table.timestamps(true, true); |
||||
|
||||
// TODO - add indexes
|
||||
}); |
||||
}; |
||||
|
||||
const down = async (knex: Knex) => { |
||||
await knex.schema.dropTable(MetaTable.JOBS); |
||||
}; |
||||
|
||||
export { up, down }; |
||||
@ -0,0 +1,136 @@
|
||||
import type { NcContext } from '~/interface/config'; |
||||
import type { Condition } from '~/db/CustomKnex'; |
||||
import Noco from '~/Noco'; |
||||
import { |
||||
CacheDelDirection, |
||||
CacheGetType, |
||||
CacheScope, |
||||
MetaTable, |
||||
} from '~/utils/globals'; |
||||
import NocoCache from '~/cache/NocoCache'; |
||||
import { extractProps } from '~/helpers/extractProps'; |
||||
import { prepareForDb, prepareForResponse } from '~/utils/modelUtils'; |
||||
|
||||
export default class Job { |
||||
id: string; |
||||
job: string; |
||||
status: string; |
||||
result: string; |
||||
fk_user_id: string; |
||||
fk_workspace_id: string; |
||||
base_id: string; |
||||
created_at: Date; |
||||
updated_at: Date; |
||||
|
||||
constructor(data: Partial<Job>) { |
||||
Object.assign(this, data); |
||||
} |
||||
|
||||
public static async insert( |
||||
context: NcContext, |
||||
jobObj: Partial<Job>, |
||||
ncMeta = Noco.ncMeta, |
||||
) { |
||||
const insertObj = extractProps(jobObj, [ |
||||
'job', |
||||
'status', |
||||
'result', |
||||
'fk_user_id', |
||||
]); |
||||
|
||||
const { id } = await ncMeta.metaInsert2( |
||||
context.workspace_id, |
||||
context.base_id, |
||||
MetaTable.JOBS, |
||||
insertObj, |
||||
); |
||||
|
||||
return this.get(context, id, ncMeta); |
||||
} |
||||
|
||||
public static async update( |
||||
context: NcContext, |
||||
jobId: string, |
||||
jobObj: Partial<Job>, |
||||
ncMeta = Noco.ncMeta, |
||||
) { |
||||
const updateObj = extractProps(jobObj, ['status', 'result']); |
||||
|
||||
const res = await ncMeta.metaUpdate( |
||||
context.workspace_id, |
||||
context.base_id, |
||||
MetaTable.JOBS, |
||||
prepareForDb(updateObj, 'result'), |
||||
jobId, |
||||
); |
||||
|
||||
await NocoCache.update( |
||||
`${CacheScope.JOBS}:${jobId}`, |
||||
prepareForResponse(updateObj, 'result'), |
||||
); |
||||
|
||||
return res; |
||||
} |
||||
|
||||
public static async delete( |
||||
context: NcContext, |
||||
jobId: string, |
||||
ncMeta = Noco.ncMeta, |
||||
) { |
||||
await ncMeta.metaDelete( |
||||
context.workspace_id, |
||||
context.base_id, |
||||
MetaTable.JOBS, |
||||
jobId, |
||||
); |
||||
|
||||
await NocoCache.deepDel( |
||||
`${CacheScope.JOBS}:${jobId}`, |
||||
CacheDelDirection.CHILD_TO_PARENT, |
||||
); |
||||
} |
||||
|
||||
public static async get(context: NcContext, id: any, ncMeta = Noco.ncMeta) { |
||||
let jobData = |
||||
id && |
||||
(await NocoCache.get( |
||||
`${CacheScope.JOBS}:${id}`, |
||||
CacheGetType.TYPE_OBJECT, |
||||
)); |
||||
|
||||
if (!jobData) { |
||||
jobData = await ncMeta.metaGet2( |
||||
context.workspace_id, |
||||
context.base_id, |
||||
MetaTable.JOBS, |
||||
id, |
||||
); |
||||
|
||||
jobData = prepareForResponse(jobData, 'result'); |
||||
|
||||
await NocoCache.set(`${CacheScope.JOBS}:${id}`, jobData); |
||||
} |
||||
|
||||
return jobData && new Job(jobData); |
||||
} |
||||
|
||||
public static async list( |
||||
context: NcContext, |
||||
opts: { |
||||
condition?: Record<string, string>; |
||||
xcCondition?: Condition; |
||||
}, |
||||
ncMeta = Noco.ncMeta, |
||||
): Promise<Job[]> { |
||||
const jobList = await ncMeta.metaList2( |
||||
context.workspace_id, |
||||
context.base_id, |
||||
MetaTable.JOBS, |
||||
opts, |
||||
); |
||||
|
||||
return jobList.map((job) => { |
||||
return new Job(prepareForResponse(job, 'result')); |
||||
}); |
||||
} |
||||
} |
||||
@ -1,54 +0,0 @@
|
||||
import { |
||||
OnQueueActive, |
||||
OnQueueCompleted, |
||||
OnQueueFailed, |
||||
Processor, |
||||
} from '@nestjs/bull'; |
||||
import { Job } from 'bull'; |
||||
import { EventEmitter2 } from '@nestjs/event-emitter'; |
||||
import { Logger } from '@nestjs/common'; |
||||
import { JobEvents, JOBS_QUEUE, JobStatus } from '~/interface/Jobs'; |
||||
|
||||
@Processor(JOBS_QUEUE) |
||||
export class JobsEventService { |
||||
protected logger = new Logger(JobsEventService.name); |
||||
|
||||
constructor(private eventEmitter: EventEmitter2) {} |
||||
|
||||
@OnQueueActive() |
||||
onActive(job: Job) { |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.ACTIVE, |
||||
}); |
||||
} |
||||
|
||||
@OnQueueFailed() |
||||
onFailed(job: Job, error: Error) { |
||||
this.logger.error( |
||||
`---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, |
||||
); |
||||
|
||||
const newLocal = this; |
||||
newLocal.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.FAILED, |
||||
data: { |
||||
error: { |
||||
message: error?.message, |
||||
}, |
||||
}, |
||||
}); |
||||
} |
||||
|
||||
@OnQueueCompleted() |
||||
onCompleted(job: Job, data: any) { |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.COMPLETED, |
||||
data: { |
||||
result: data, |
||||
}, |
||||
}); |
||||
} |
||||
} |
||||
@ -0,0 +1,108 @@
|
||||
import { |
||||
OnQueueActive, |
||||
OnQueueCompleted, |
||||
OnQueueFailed, |
||||
Processor, |
||||
} from '@nestjs/bull'; |
||||
import { Job as BullJob } from 'bull'; |
||||
import { EventEmitter2 } from '@nestjs/event-emitter'; |
||||
import { Logger } from '@nestjs/common'; |
||||
import { JobEvents, JOBS_QUEUE, JobStatus } from '~/interface/Jobs'; |
||||
import { Job } from '~/models'; |
||||
import { RootScopes } from '~/utils/globals'; |
||||
|
||||
@Processor(JOBS_QUEUE) |
||||
export class JobsEventService { |
||||
protected logger = new Logger(JobsEventService.name); |
||||
|
||||
constructor(private eventEmitter: EventEmitter2) {} |
||||
|
||||
@OnQueueActive() |
||||
onActive(job: BullJob) { |
||||
Job.update( |
||||
{ |
||||
workspace_id: RootScopes.ROOT, |
||||
base_id: RootScopes.ROOT, |
||||
}, |
||||
job.id.toString(), |
||||
{ |
||||
status: JobStatus.ACTIVE, |
||||
}, |
||||
) |
||||
.then(() => { |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.ACTIVE, |
||||
}); |
||||
}) |
||||
.catch((error) => { |
||||
this.logger.error( |
||||
`Failed to update job (${job.id}) status to active: ${error.message}`, |
||||
); |
||||
}); |
||||
} |
||||
|
||||
@OnQueueFailed() |
||||
onFailed(job: BullJob, error: Error) { |
||||
this.logger.error( |
||||
`---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, |
||||
); |
||||
|
||||
Job.update( |
||||
{ |
||||
workspace_id: RootScopes.ROOT, |
||||
base_id: RootScopes.ROOT, |
||||
}, |
||||
job.id.toString(), |
||||
{ |
||||
status: JobStatus.FAILED, |
||||
}, |
||||
) |
||||
.then(() => { |
||||
const newLocal = this; |
||||
newLocal.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.FAILED, |
||||
data: { |
||||
error: { |
||||
message: error?.message, |
||||
}, |
||||
}, |
||||
}); |
||||
}) |
||||
.catch((error) => { |
||||
this.logger.error( |
||||
`Failed to update job (${job.id}) status to failed: ${error.message}`, |
||||
); |
||||
}); |
||||
} |
||||
|
||||
@OnQueueCompleted() |
||||
onCompleted(job: BullJob, data: any) { |
||||
Job.update( |
||||
{ |
||||
workspace_id: RootScopes.ROOT, |
||||
base_id: RootScopes.ROOT, |
||||
}, |
||||
job.id.toString(), |
||||
{ |
||||
status: JobStatus.COMPLETED, |
||||
result: data, |
||||
}, |
||||
) |
||||
.then(() => { |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.COMPLETED, |
||||
data: { |
||||
result: data, |
||||
}, |
||||
}); |
||||
}) |
||||
.catch((error) => { |
||||
this.logger.error( |
||||
`Failed to update job (${job.id}) status to completed: ${error.message}`, |
||||
); |
||||
}); |
||||
} |
||||
} |
||||
@ -0,0 +1,58 @@
|
||||
import { |
||||
Body, |
||||
Controller, |
||||
HttpCode, |
||||
Inject, |
||||
Param, |
||||
Post, |
||||
Req, |
||||
UseGuards, |
||||
} from '@nestjs/common'; |
||||
import type { DataExportJobData } from '~/interface/Jobs'; |
||||
import { GlobalGuard } from '~/guards/global/global.guard'; |
||||
import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware'; |
||||
import { BasesService } from '~/services/bases.service'; |
||||
import { View } from '~/models'; |
||||
import { JobTypes } from '~/interface/Jobs'; |
||||
import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard'; |
||||
import { IJobsService } from '~/modules/jobs/jobs-service.interface'; |
||||
import { TenantContext } from '~/decorators/tenant-context.decorator'; |
||||
import { NcContext, NcRequest } from '~/interface/config'; |
||||
import { NcError } from '~/helpers/catchError'; |
||||
|
||||
@Controller() |
||||
@UseGuards(MetaApiLimiterGuard, GlobalGuard) |
||||
export class DataExportController { |
||||
constructor( |
||||
@Inject('JobsService') protected readonly jobsService: IJobsService, |
||||
protected readonly basesService: BasesService, |
||||
) {} |
||||
|
||||
@Post(['/api/v2/export/:viewId/:exportAs']) |
||||
@HttpCode(200) |
||||
// TODO add new ACL
|
||||
@Acl('dataList') |
||||
async exportModelData( |
||||
@TenantContext() context: NcContext, |
||||
@Req() req: NcRequest, |
||||
@Param('viewId') viewId: string, |
||||
@Param('exportAs') exportAs: 'csv' | 'json' | 'xlsx', |
||||
@Body() options: DataExportJobData['options'], |
||||
) { |
||||
const view = await View.get(context, viewId); |
||||
|
||||
if (!view) NcError.viewNotFound(viewId); |
||||
|
||||
const job = await this.jobsService.add(JobTypes.DataExport, { |
||||
context, |
||||
options, |
||||
modelId: view.fk_model_id, |
||||
viewId, |
||||
user: req.user, |
||||
exportAs, |
||||
ncSiteUrl: req.ncSiteUrl, |
||||
}); |
||||
|
||||
return job; |
||||
} |
||||
} |
||||
@ -0,0 +1,132 @@
|
||||
import { Readable } from 'stream'; |
||||
import path from 'path'; |
||||
import { Process, Processor } from '@nestjs/bull'; |
||||
import { Logger } from '@nestjs/common'; |
||||
import { Job } from 'bull'; |
||||
import moment from 'moment'; |
||||
import { type DataExportJobData, JOBS_QUEUE, JobTypes } from '~/interface/Jobs'; |
||||
import { elapsedTime, initTime } from '~/modules/jobs/helpers'; |
||||
import { ExportService } from '~/modules/jobs/jobs/export-import/export.service'; |
||||
import { Model, PresignedUrl, View } from '~/models'; |
||||
import { NcError } from '~/helpers/catchError'; |
||||
import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2'; |
||||
|
||||
function getViewTitle(view: View) { |
||||
return view.is_default ? 'Default View' : view.title; |
||||
} |
||||
|
||||
@Processor(JOBS_QUEUE) |
||||
export class DataExportProcessor { |
||||
private logger = new Logger(DataExportProcessor.name); |
||||
|
||||
constructor(private readonly exportService: ExportService) {} |
||||
|
||||
@Process(JobTypes.DataExport) |
||||
async job(job: Job<DataExportJobData>) { |
||||
const { |
||||
context, |
||||
options, |
||||
modelId, |
||||
viewId, |
||||
user: _user, |
||||
exportAs, |
||||
ncSiteUrl, |
||||
} = job.data; |
||||
|
||||
if (exportAs !== 'csv') NcError.notImplemented(`Export as ${exportAs}`); |
||||
|
||||
const hrTime = initTime(); |
||||
|
||||
const model = await Model.get(context, modelId); |
||||
|
||||
if (!model) NcError.tableNotFound(modelId); |
||||
|
||||
const view = await View.get(context, viewId); |
||||
|
||||
if (!view) NcError.viewNotFound(viewId); |
||||
|
||||
// date time as containing folder YYYY-MM-DD/HH
|
||||
const dateFolder = moment().format('YYYY-MM-DD/HH'); |
||||
|
||||
const storageAdapter = await NcPluginMgrv2.storageAdapter(); |
||||
|
||||
const destPath = `nc/uploads/data-export/${dateFolder}/${modelId}/${ |
||||
model.title |
||||
} (${getViewTitle(view)}) - ${Date.now()}.csv`;
|
||||
|
||||
let url = null; |
||||
|
||||
try { |
||||
const dataStream = new Readable({ |
||||
read() {}, |
||||
}); |
||||
|
||||
dataStream.setEncoding('utf8'); |
||||
|
||||
let error = null; |
||||
|
||||
const uploadFilePromise = (storageAdapter as any) |
||||
.fileCreateByStream(destPath, dataStream) |
||||
.catch((e) => { |
||||
this.logger.error(e); |
||||
error = e; |
||||
}); |
||||
|
||||
this.exportService |
||||
.streamModelDataAsCsv(context, { |
||||
dataStream, |
||||
linkStream: null, |
||||
baseId: model.base_id, |
||||
modelId: model.id, |
||||
viewId: view.id, |
||||
ncSiteUrl: ncSiteUrl, |
||||
delimiter: options?.delimiter, |
||||
}) |
||||
.catch((e) => { |
||||
this.logger.debug(e); |
||||
dataStream.push(null); |
||||
error = e; |
||||
}); |
||||
|
||||
url = await uploadFilePromise; |
||||
|
||||
// if url is not defined, it is local attachment
|
||||
if (!url) { |
||||
url = await PresignedUrl.getSignedUrl({ |
||||
path: path.join(destPath.replace('nc/uploads/', '')), |
||||
filename: `${model.title} (${getViewTitle(view)}).csv`, |
||||
expireSeconds: 3 * 60 * 60, // 3 hours
|
||||
}); |
||||
} else { |
||||
if (url.includes('.amazonaws.com/')) { |
||||
const relativePath = decodeURI(url.split('.amazonaws.com/')[1]); |
||||
url = await PresignedUrl.getSignedUrl({ |
||||
path: relativePath, |
||||
filename: `${model.title} (${getViewTitle(view)}).csv`, |
||||
s3: true, |
||||
expireSeconds: 3 * 60 * 60, // 3 hours
|
||||
}); |
||||
} |
||||
} |
||||
|
||||
if (error) { |
||||
throw error; |
||||
} |
||||
|
||||
elapsedTime( |
||||
hrTime, |
||||
`exported data for model ${modelId} view ${viewId} as ${exportAs}`, |
||||
'exportData', |
||||
); |
||||
} catch (e) { |
||||
throw NcError.badRequest(e); |
||||
} |
||||
|
||||
return { |
||||
timestamp: new Date(), |
||||
type: exportAs, |
||||
title: `${model.title} (${getViewTitle(view)})`, |
||||
url, |
||||
}; |
||||
} |
||||
} |
||||
@ -1,53 +0,0 @@
|
||||
import { |
||||
OnQueueActive, |
||||
OnQueueCompleted, |
||||
OnQueueFailed, |
||||
Processor, |
||||
} from '@nestjs/bull'; |
||||
import { Job } from 'bull'; |
||||
import { EventEmitter2 } from '@nestjs/event-emitter'; |
||||
import { Logger } from '@nestjs/common'; |
||||
import { JobEvents, JOBS_QUEUE, JobStatus } from '~/interface/Jobs'; |
||||
|
||||
@Processor(JOBS_QUEUE) |
||||
export class JobsEventService { |
||||
protected logger = new Logger(JobsEventService.name); |
||||
|
||||
constructor(private eventEmitter: EventEmitter2) {} |
||||
|
||||
@OnQueueActive() |
||||
onActive(job: Job) { |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.ACTIVE, |
||||
}); |
||||
} |
||||
|
||||
@OnQueueFailed() |
||||
onFailed(job: Job, error: Error) { |
||||
this.logger.error( |
||||
`---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, |
||||
); |
||||
|
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.FAILED, |
||||
data: { |
||||
error: { |
||||
message: error?.message, |
||||
}, |
||||
}, |
||||
}); |
||||
} |
||||
|
||||
@OnQueueCompleted() |
||||
onCompleted(job: Job, data: any) { |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.COMPLETED, |
||||
data: { |
||||
result: data, |
||||
}, |
||||
}); |
||||
} |
||||
} |
||||
@ -0,0 +1,19 @@
|
||||
import { Test } from '@nestjs/testing'; |
||||
import { JobsMetaService } from './jobs-meta.service'; |
||||
import type { TestingModule } from '@nestjs/testing'; |
||||
|
||||
describe('JobsMetaService', () => { |
||||
let service: JobsMetaService; |
||||
|
||||
beforeEach(async () => { |
||||
const module: TestingModule = await Test.createTestingModule({ |
||||
providers: [JobsMetaService], |
||||
}).compile(); |
||||
|
||||
service = module.get<JobsMetaService>(JobsMetaService); |
||||
}); |
||||
|
||||
it('should be defined', () => { |
||||
expect(service).toBeDefined(); |
||||
}); |
||||
}); |
||||
@ -0,0 +1,83 @@
|
||||
import { Injectable } from '@nestjs/common'; |
||||
import dayjs from 'dayjs'; |
||||
import type { NcContext, NcRequest } from '~/interface/config'; |
||||
import type { JobTypes } from '~/interface/Jobs'; |
||||
import { JobStatus } from '~/interface/Jobs'; |
||||
import { Job } from '~/models'; |
||||
import Noco from '~/Noco'; |
||||
|
||||
@Injectable() |
||||
export class JobsMetaService { |
||||
constructor() {} |
||||
|
||||
async list( |
||||
context: NcContext, |
||||
param: { job?: JobTypes; status?: JobStatus }, |
||||
req: NcRequest, |
||||
) { |
||||
/* |
||||
* List jobs for the current base. |
||||
* If the job is not created by the current user, exclude the result. |
||||
* List jobs updated in the last 1 hour or jobs that are still active(, waiting, or delayed). |
||||
*/ |
||||
return Job.list(context, { |
||||
xcCondition: { |
||||
_and: [ |
||||
...(param.job |
||||
? [ |
||||
{ |
||||
job: { |
||||
eq: param.job, |
||||
}, |
||||
}, |
||||
] |
||||
: []), |
||||
...(param.status |
||||
? [ |
||||
{ |
||||
status: { |
||||
eq: param.status, |
||||
}, |
||||
}, |
||||
] |
||||
: []), |
||||
{ |
||||
_or: [ |
||||
{ |
||||
updated_at: { |
||||
gt: Noco.ncMeta.formatDateTime( |
||||
dayjs().subtract(1, 'hour').toISOString(), |
||||
), |
||||
}, |
||||
}, |
||||
{ |
||||
status: { |
||||
eq: JobStatus.ACTIVE, |
||||
}, |
||||
}, |
||||
{ |
||||
status: { |
||||
eq: JobStatus.WAITING, |
||||
}, |
||||
}, |
||||
{ |
||||
status: { |
||||
eq: JobStatus.DELAYED, |
||||
}, |
||||
}, |
||||
], |
||||
}, |
||||
], |
||||
}, |
||||
}).then((jobs) => { |
||||
return jobs.map((job) => { |
||||
if (job.fk_user_id === req.user.id) { |
||||
return job; |
||||
} else { |
||||
const { result, ...rest } = job; |
||||
return rest; |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue