From f4cca91d13affb7f310b5ec47f2d35eb668ad650 Mon Sep 17 00:00:00 2001 From: Louis Delbosc Date: Wed, 27 Jul 2022 12:13:16 +0200 Subject: [PATCH] feat: add export excel on public views --- .../spreadsheet/components/MoreActions.vue | 75 ++++++++---- .../src/lib/meta/api/dataApis/helpers.ts | 86 ++++---------- .../api/publicApis/publicDataExportApis.ts | 109 ++++++++++++------ 3 files changed, 153 insertions(+), 117 deletions(-) diff --git a/packages/nc-gui/components/project/spreadsheet/components/MoreActions.vue b/packages/nc-gui/components/project/spreadsheet/components/MoreActions.vue index 549e0232b0..6e6ec7df43 100644 --- a/packages/nc-gui/components/project/spreadsheet/components/MoreActions.vue +++ b/packages/nc-gui/components/project/spreadsheet/components/MoreActions.vue @@ -233,27 +233,62 @@ export default { async exportExcel() { let offset = 0; let c = 1; - const res = await this.$api.dbViewRow.export( - 'noco', - this.projectName, - this.meta.title, - this.selectedView.title, - ExportTypes.EXCEL, - { - responseType: 'base64', - query: { - offset, - }, - } - ); - const workbook = XLSX.read(res.data, { type: 'base64' }); - XLSX.writeFile(workbook, `${this.meta.title}_exported_${c++}.xlsx`); + try { + while (!isNaN(offset) && offset > -1) { + let res; + if (this.publicViewId) { + console.log('IF', this.publicViewId) + res = await this.$api.public.csvExport(this.publicViewId, ExportTypes.EXCEL, { + responseType: 'blob', + query: { + fields: + this.queryParams && + this.queryParams.fieldsOrder && + this.queryParams.fieldsOrder.filter(c => this.queryParams.showFields[c]), + offset, + sortArrJson: JSON.stringify( + this.reqPayload && + this.reqPayload.sorts && + this.reqPayload.sorts.map(({ fk_column_id, direction }) => ({ + direction, + fk_column_id, + })) + ), + filterArrJson: JSON.stringify(this.reqPayload && this.reqPayload.filters), + }, + headers: { + 'xc-password': this.reqPayload && this.reqPayload.password, + }, + }); + } else { + console.log('ELSE') + res = await this.$api.dbViewRow.export( + 'noco', + this.projectName, + this.meta.title, + this.selectedView.title, + ExportTypes.EXCEL, + { + responseType: 'base64', + query: { + offset, + }, + } + ); + } + const workbook = XLSX.read(res.data, { type: 'base64' }); + XLSX.writeFile(workbook, `${this.meta.title}_exported_${c++}.xlsx`); - offset = +res.headers['nc-export-offset']; - if (offset > -1) { - this.$toast.info('Downloading more files').goAway(3000); - } else { - this.$toast.success('Successfully exported all table data').goAway(3000); + offset = +res.headers['nc-export-offset']; + if (offset > -1) { + this.$toast.info('Downloading more files').goAway(3000); + } else { + this.$toast.success('Successfully exported all table data').goAway(3000); + } + } + } catch (e) { + console.log(e); + this.$toast.error(e.message).goAway(3000); } }, async exportCsv() { diff --git a/packages/nocodb/src/lib/meta/api/dataApis/helpers.ts b/packages/nocodb/src/lib/meta/api/dataApis/helpers.ts index 19952ee9a1..968f062265 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/helpers.ts +++ b/packages/nocodb/src/lib/meta/api/dataApis/helpers.ts @@ -57,53 +57,10 @@ export async function extractXlsxData(view: View, req: Request) { dbDriver: NcConnectionMgrv2.get(base), }); - let offset = +req.query.offset || 0; - const limit = 100; - // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; - const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; - const csvRows = []; - const startTime = process.hrtime(); - let elapsed, temp; - - for ( - elapsed = 0; - elapsed < timeout; - offset += limit, - temp = process.hrtime(startTime), - elapsed = temp[0] * 1000 + temp[1] / 1000000 - ) { - const rows = await nocoExecute( - await getAst({ - query: req.query, - includePkByDefault: false, - model: view.model, - view, - }), - await baseModel.list({ ...req.query, offset, limit }), - {}, - req.query - ); - - if (!rows?.length) { - offset = -1; - break; - } - - for (const row of rows) { - const csvRow = { ...row }; + const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req); + const data = XLSX.utils.json_to_sheet(dbRows); - for (const column of view.model.columns) { - if (isSystemColumn(column) && !view.show_system_fields) continue; - csvRow[column.title] = await serializeCellValue({ - value: row[column.title], - column, - }); - } - csvRows.push(csvRow); - } - } - const data = XLSX.utils.json_to_sheet(csvRows); - return { offset, csvRows, elapsed, data }; + return { offset, dbRows, elapsed, data }; } export async function extractCsvData(view: View, req: Request) { @@ -126,11 +83,27 @@ export async function extractCsvData(view: View, req: Request) { dbDriver: NcConnectionMgrv2.get(base), }); + const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req); + + const data = papaparse.unparse( + { + fields: view.model.columns.map((c) => c.title), + data: dbRows, + }, + { + escapeFormulae: true, + } + ); + + return { offset, dbRows, elapsed, data }; +} + +async function getDbRows(baseModel, view: View, req: Request) { let offset = +req.query.offset || 0; const limit = 100; // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; - const csvRows = []; + const dbRows = []; const startTime = process.hrtime(); let elapsed, temp; @@ -159,30 +132,19 @@ export async function extractCsvData(view: View, req: Request) { } for (const row of rows) { - const csvRow = { ...row }; + const dbRow = { ...row }; for (const column of view.model.columns) { if (isSystemColumn(column) && !view.show_system_fields) continue; - csvRow[column.title] = await serializeCellValue({ + dbRow[column.title] = await serializeCellValue({ value: row[column.title], column, }); } - csvRows.push(csvRow); + dbRows.push(dbRow); } } - - const data = papaparse.unparse( - { - fields: view.model.columns.map((c) => c.title), - data: csvRows, - }, - { - escapeFormulae: true, - } - ); - - return { offset, csvRows, elapsed, data }; + return { offset, dbRows, elapsed }; } export async function serializeCellValue({ diff --git a/packages/nocodb/src/lib/meta/api/publicApis/publicDataExportApis.ts b/packages/nocodb/src/lib/meta/api/publicApis/publicDataExportApis.ts index 49ecdf394a..f5ed20d574 100644 --- a/packages/nocodb/src/lib/meta/api/publicApis/publicDataExportApis.ts +++ b/packages/nocodb/src/lib/meta/api/publicApis/publicDataExportApis.ts @@ -1,4 +1,5 @@ import { Request, Response, Router } from 'express'; +import * as XLSX from 'xlsx'; import View from '../../../models/View'; import Model from '../../../models/Model'; import Base from '../../../models/Base'; @@ -12,8 +13,38 @@ import LookupColumn from '../../../models/LookupColumn'; import catchError, { NcError } from '../../helpers/catchError'; import getAst from '../../../db/sql-data-mapper/lib/sql/helpers/getAst'; +async function exportExcel(req: Request, res: Response) { + const view = await View.getByUUID(req.params.publicDataUuid); + if (!view) NcError.notFound('Not found'); + if (view.type !== ViewTypes.GRID) NcError.notFound('Not found'); + + if (view.password && view.password !== req.headers?.['xc-password']) { + NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const model = await view.getModelWithInfo(); + await view.getColumns(); + + const { offset, dbRows, elapsed } = await getDbRows(model, view, req); + + const data = XLSX.utils.json_to_sheet(dbRows); + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, data, view.title); + const buf = XLSX.write(wb, { type: "base64", bookType: "xlsx" }); + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + view.title + )}-export.xlsx"`, + }); + res.end(buf); +} + async function exportCsv(req: Request, res: Response) { const view = await View.getByUUID(req.params.publicDataUuid); + const fields = req.query.fields; if (!view) NcError.notFound('Not found'); if (view.type !== ViewTypes.GRID) NcError.notFound('Not found'); @@ -25,6 +56,40 @@ async function exportCsv(req: Request, res: Response) { const model = await view.getModelWithInfo(); await view.getColumns(); + const { offset, dbRows, elapsed } = await getDbRows(model, view, req); + + const data = papaparse.unparse( + { + fields: model.columns + .sort((c1, c2) => + Array.isArray(fields) + ? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any) + : 0 + ) + .filter( + (c) => + !fields || !Array.isArray(fields) || fields.includes(c.title as any) + ) + .map((c) => c.title), + data: dbRows, + }, + { + escapeFormulae: true, + } + ); + + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + view.title + )}-export.csv"`, + }); + res.send(data); +} + +async function getDbRows(model, view: View, req: Request) { view.model.columns = view.columns .filter((c) => c.show) .map( @@ -35,7 +100,6 @@ async function exportCsv(req: Request, res: Response) { if (!model) NcError.notFound('Table not found'); - const fields = req.query.fields; const listArgs: any = { ...req.query }; try { listArgs.filterArr = JSON.parse(listArgs.filterArrJson); @@ -62,7 +126,7 @@ async function exportCsv(req: Request, res: Response) { const limit = 100; // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; - const csvRows = []; + const dbRows = []; const startTime = process.hrtime(); let elapsed, temp; @@ -86,47 +150,18 @@ async function exportCsv(req: Request, res: Response) { } for (const row of rows) { - const csvRow = { ...row }; + const dbRow = { ...row }; for (const column of view.model.columns) { - csvRow[column.title] = await serializeCellValue({ + dbRow[column.title] = await serializeCellValue({ value: row[column.title], column, }); } - csvRows.push(csvRow); + dbRows.push(dbRow); } } - - const data = papaparse.unparse( - { - fields: model.columns - .sort((c1, c2) => - Array.isArray(fields) - ? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any) - : 0 - ) - .filter( - (c) => - !fields || !Array.isArray(fields) || fields.includes(c.title as any) - ) - .map((c) => c.title), - data: csvRows, - }, - { - escapeFormulae: true, - } - ); - - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - view.title - )}-export.csv"`, - }); - res.send(data); + return { offset, dbRows, elapsed }; } async function serializeCellValue({ @@ -198,4 +233,8 @@ router.get( '/api/v1/db/public/shared-view/:publicDataUuid/rows/export/csv', catchError(exportCsv) ); +router.get( + '/api/v1/db/public/shared-view/:publicDataUuid/rows/export/excel', + catchError(exportExcel) +); export default router;