From bd7861353afa8afe3697b1278cac60927dbb5d41 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 10 Apr 2023 00:39:19 +0530 Subject: [PATCH] feat: public shared data export apis Signed-off-by: Pranav C --- packages/nocodb-nest/src/app.module.ts | 3 +- .../public-datas-export.controller.spec.ts | 20 ++ .../public-datas-export.controller.ts | 280 ++++++++++++++++++ .../public-datas-export.module.ts | 9 + .../public-datas-export.service.spec.ts | 18 ++ .../public-datas-export.service.ts | 4 + 6 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.controller.spec.ts create mode 100644 packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.controller.ts create mode 100644 packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.module.ts create mode 100644 packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.service.spec.ts create mode 100644 packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.service.ts diff --git a/packages/nocodb-nest/src/app.module.ts b/packages/nocodb-nest/src/app.module.ts index 3cbcbc9174..419bd32804 100644 --- a/packages/nocodb-nest/src/app.module.ts +++ b/packages/nocodb-nest/src/app.module.ts @@ -40,9 +40,10 @@ import { DataAliasModule } from './modules/data-alias/data-alias.module'; import { ApiDocsModule } from './modules/api-docs/api-docs.module'; import { PublicMetasModule } from './modules/public-metas/public-metas.module'; import { PublicDatasModule } from './modules/public-datas/public-datas.module'; +import { PublicDatasExportModule } from './modules/public-datas-export/public-datas-export.module'; @Module({ - imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule, AuditsModule, DatasModule, DataAliasModule, ApiDocsModule, PublicMetasModule, PublicDatasModule], + imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule, AuditsModule, DatasModule, DataAliasModule, ApiDocsModule, PublicMetasModule, PublicDatasModule, PublicDatasExportModule], controllers: [], providers: [Connection, MetaService, JwtStrategy, ExtractProjectIdMiddleware], exports: [Connection, MetaService], diff --git a/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.controller.spec.ts b/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.controller.spec.ts new file mode 100644 index 0000000000..1a66b54084 --- /dev/null +++ b/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { PublicDatasExportController } from './public-datas-export.controller'; +import { PublicDatasExportService } from './public-datas-export.service'; + +describe('PublicDatasExportController', () => { + let controller: PublicDatasExportController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [PublicDatasExportController], + providers: [PublicDatasExportService], + }).compile(); + + controller = module.get(PublicDatasExportController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.controller.ts b/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.controller.ts new file mode 100644 index 0000000000..ce7edb120e --- /dev/null +++ b/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.controller.ts @@ -0,0 +1,280 @@ +import { Controller, Get, Request, Param, Response } from '@nestjs/common'; +import { ErrorMessages, isSystemColumn, UITypes, ViewTypes } from 'nocodb-sdk'; +import { NcError } from '../../helpers/catchError'; +import getAst from '../../helpers/getAst'; +import { + Base, + Column, + LinkToAnotherRecordColumn, + LookupColumn, + Model, + View, +} from '../../models'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { PublicDatasExportService } from './public-datas-export.service'; +import * as XLSX from 'xlsx'; +import { nocoExecute } from 'nc-help'; +import papaparse from 'papaparse'; +import { serializeCellValue } from '../datas/helpers'; + +@Controller('public-datas-export') +export class PublicDatasExportController { + constructor( + private readonly publicDatasExportService: PublicDatasExportService, + ) {} + + @Get('/api/v1/db/public/shared-view/:publicDataUuid/rows/export/csv') + async exportExcel( + @Request() req, + @Response() res, + @Param('publicDataUuid') publicDataUuid: string, + ) { + const view = await View.getByUUID(publicDataUuid); + if (!view) NcError.notFound('Not found'); + if ( + view.type !== ViewTypes.GRID && + view.type !== ViewTypes.KANBAN && + view.type !== ViewTypes.GALLERY && + view.type !== ViewTypes.MAP + ) + NcError.notFound('Not found'); + + if (view.password && view.password !== req.headers?.['xc-password']) { + NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const model = await view.getModelWithInfo(); + + await view.getColumns(); + + const { offset, dbRows, elapsed } = await this.getDbRows(model, view, req); + + const fields = req.query.fields as string[]; + + const data = XLSX.utils.json_to_sheet( + dbRows.map((o: Record) => + Object.fromEntries(fields.map((f) => [f, o[f]])), + ), + { header: fields }, + ); + + const wb = XLSX.utils.book_new(); + + XLSX.utils.book_append_sheet(wb, data, view.title); + + const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); + + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + view.title, + )}-export.xlsx"`, + }); + res.end(buf); + } + + @Get('/api/v1/db/public/shared-view/:publicDataUuid/rows/export/excel') + async exportCsv(@Request() req, @Response() res) { + const view = await View.getByUUID(req.params.publicDataUuid); + const fields = req.query.fields; + + if (!view) NcError.notFound('Not found'); + if ( + view.type !== ViewTypes.GRID && + view.type !== ViewTypes.KANBAN && + view.type !== ViewTypes.GALLERY && + view.type !== ViewTypes.MAP + ) + NcError.notFound('Not found'); + + if (view.password && view.password !== req.headers?.['xc-password']) { + NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const model = await view.getModelWithInfo(); + await view.getColumns(); + + const { offset, dbRows, elapsed } = await this.getDbRows(model, view, req); + + const data = papaparse.unparse( + { + fields: model.columns + .sort((c1, c2) => + Array.isArray(fields) + ? fields.indexOf(c1.title as any) - + fields.indexOf(c2.title as any) + : 0, + ) + .filter( + (c) => + !fields || + !Array.isArray(fields) || + fields.includes(c.title as any), + ) + .map((c) => c.title), + data: dbRows, + }, + { + escapeFormulae: true, + }, + ); + + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + view.title, + )}-export.csv"`, + }); + res.send(data); + } + + async getDbRows(model, view: View, req) { + view.model.columns = view.columns + .filter((c) => c.show) + .map( + (c) => + new Column({ + ...c, + ...view.model.columnsById[c.fk_column_id], + } as any), + ) + .filter((column) => !isSystemColumn(column) || view.show_system_fields); + + if (!model) NcError.notFound('Table not found'); + + const listArgs: any = { ...req.query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + + const base = await Base.get(model.base_id); + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: await NcConnectionMgrv2.get(base), + }); + + const { ast } = await getAst({ + query: req.query, + model, + view, + includePkByDefault: false, + }); + + let offset = +req.query.offset || 0; + const limit = 100; + // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; + const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; + const dbRows = []; + const startTime = process.hrtime(); + let elapsed, temp; + + for ( + elapsed = 0; + elapsed < timeout; + offset += limit, + temp = process.hrtime(startTime), + elapsed = temp[0] * 1000 + temp[1] / 1000000 + ) { + const rows = await nocoExecute( + ast, + await baseModel.list({ ...listArgs, offset, limit }), + {}, + listArgs, + ); + + if (!rows?.length) { + offset = -1; + break; + } + + for (const row of rows) { + const dbRow = { ...row }; + + for (const column of view.model.columns) { + dbRow[column.title] = await serializeCellValue({ + value: row[column.title], + column, + siteUrl: req.ncSiteUrl, + }); + } + dbRows.push(dbRow); + } + } + return { offset, dbRows, elapsed }; + } + + async serializeCellValue({ + value, + column, + ncSiteUrl, + }: { + column?: Column; + value: any; + ncSiteUrl?: string; + }) { + if (!column) { + return value; + } + + if (!value) return value; + + switch (column?.uidt) { + case UITypes.Attachment: { + let data = value; + try { + if (typeof value === 'string') { + data = JSON.parse(value); + } + } catch {} + + return (data || []).map( + (attachment) => + `${encodeURI(attachment.title)}(${encodeURI(attachment.url)})`, + ); + } + case UITypes.Lookup: + { + const colOptions = await column.getColOptions(); + const lookupColumn = await colOptions.getLookupColumn(); + return ( + await Promise.all( + [...(Array.isArray(value) ? value : [value])].map(async (v) => + serializeCellValue({ + value: v, + column: lookupColumn, + siteUrl: ncSiteUrl, + }), + ), + ) + ).join(', '); + } + break; + case UITypes.LinkToAnotherRecord: + { + const colOptions = + await column.getColOptions(); + const relatedModel = await colOptions.getRelatedTable(); + await relatedModel.getColumns(); + return [...(Array.isArray(value) ? value : [value])] + .map((v) => { + return v[relatedModel.displayValue?.title]; + }) + .join(', '); + } + break; + default: + if (value && typeof value === 'object') { + return JSON.stringify(value); + } + return value; + } + } +} diff --git a/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.module.ts b/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.module.ts new file mode 100644 index 0000000000..5afe1627ac --- /dev/null +++ b/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { PublicDatasExportService } from './public-datas-export.service'; +import { PublicDatasExportController } from './public-datas-export.controller'; + +@Module({ + controllers: [PublicDatasExportController], + providers: [PublicDatasExportService] +}) +export class PublicDatasExportModule {} diff --git a/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.service.spec.ts b/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.service.spec.ts new file mode 100644 index 0000000000..3e82b58297 --- /dev/null +++ b/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { PublicDatasExportService } from './public-datas-export.service'; + +describe('PublicDatasExportService', () => { + let service: PublicDatasExportService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [PublicDatasExportService], + }).compile(); + + service = module.get(PublicDatasExportService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.service.ts b/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.service.ts new file mode 100644 index 0000000000..79e84613ec --- /dev/null +++ b/packages/nocodb-nest/src/modules/public-datas-export/public-datas-export.service.ts @@ -0,0 +1,4 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class PublicDatasExportService {}