diff --git a/packages/nocodb-sdk/src/lib/globals.ts b/packages/nocodb-sdk/src/lib/globals.ts index 8c990d585e..b63424b42a 100644 --- a/packages/nocodb-sdk/src/lib/globals.ts +++ b/packages/nocodb-sdk/src/lib/globals.ts @@ -12,6 +12,7 @@ export enum RelationTypes { } export enum ExportTypes { + EXCEL = 'excel', CSV = 'csv', } diff --git a/packages/nocodb/package-lock.json b/packages/nocodb/package-lock.json index 464f750569..c7b690811c 100644 --- a/packages/nocodb/package-lock.json +++ b/packages/nocodb/package-lock.json @@ -100,7 +100,8 @@ "unique-names-generator": "^4.3.1", "uuid": "^8.2.0", "validator": "^13.1.1", - "xc-core-ts": "^0.1.0" + "xc-core-ts": "^0.1.0", + "xlsx": "^0.18.5" }, "devDependencies": { "@bitjson/npm-scripts-info": "^1.0.0", @@ -2501,6 +2502,14 @@ "integrity": "sha512-qQLMr+8o0WC4FZGQTcJiKBVC59JylcPSrTtk6usvmIDFUOCKegapy1VHQwRbFMOFyb/inzUVqHs+eMYKDM1YeQ==", "dev": true }, + "node_modules/adler-32": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", + "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -4387,6 +4396,18 @@ "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", "optional": true }, + "node_modules/cfb": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", + "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", + "dependencies": { + "adler-32": "~1.3.0", + "crc-32": "~1.2.0" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/chai": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.6.tgz", @@ -4926,6 +4947,14 @@ "node": ">=10" } }, + "node_modules/codepage": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", + "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/collection-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", @@ -9887,6 +9916,14 @@ "node": ">= 0.6" } }, + "node_modules/frac": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", + "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/fragment-cache": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", @@ -19930,6 +19967,17 @@ "node": ">= 0.6" } }, + "node_modules/ssf": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", + "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", + "dependencies": { + "frac": "~1.1.2" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", @@ -24426,6 +24474,22 @@ "node": ">= 6.4.0" } }, + "node_modules/wmf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", + "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/word": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", + "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -24582,6 +24646,26 @@ "node": ">=0.10.0" } }, + "node_modules/xlsx": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", + "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", + "dependencies": { + "adler-32": "~1.3.0", + "cfb": "~1.2.1", + "codepage": "~1.15.0", + "crc-32": "~1.2.1", + "ssf": "~0.11.2", + "wmf": "~1.0.1", + "word": "~0.3.0" + }, + "bin": { + "xlsx": "bin/xlsx.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/xml": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", @@ -26624,6 +26708,11 @@ "integrity": "sha512-qQLMr+8o0WC4FZGQTcJiKBVC59JylcPSrTtk6usvmIDFUOCKegapy1VHQwRbFMOFyb/inzUVqHs+eMYKDM1YeQ==", "dev": true }, + "adler-32": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", + "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==" + }, "agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -28156,6 +28245,15 @@ "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", "optional": true }, + "cfb": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", + "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", + "requires": { + "adler-32": "~1.3.0", + "crc-32": "~1.2.0" + } + }, "chai": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.6.tgz", @@ -28569,6 +28667,11 @@ } } }, + "codepage": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", + "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==" + }, "collection-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", @@ -32481,6 +32584,11 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, + "frac": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", + "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==" + }, "fragment-cache": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", @@ -40312,6 +40420,14 @@ "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==" }, + "ssf": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", + "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", + "requires": { + "frac": "~1.1.2" + } + }, "sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", @@ -43869,6 +43985,16 @@ "triple-beam": "^1.3.0" } }, + "wmf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", + "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==" + }, + "word": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", + "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==" + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -43985,6 +44111,20 @@ } } }, + "xlsx": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", + "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", + "requires": { + "adler-32": "~1.3.0", + "cfb": "~1.2.1", + "codepage": "~1.15.0", + "crc-32": "~1.2.1", + "ssf": "~0.11.2", + "wmf": "~1.0.1", + "word": "~0.3.0" + } + }, "xml": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", diff --git a/packages/nocodb/package.json b/packages/nocodb/package.json index d436b04bb1..147e41b876 100644 --- a/packages/nocodb/package.json +++ b/packages/nocodb/package.json @@ -186,7 +186,8 @@ "unique-names-generator": "^4.3.1", "uuid": "^8.2.0", "validator": "^13.1.1", - "xc-core-ts": "^0.1.0" + "xc-core-ts": "^0.1.0", + "xlsx": "^0.18.5" }, "devDependencies": { "@bitjson/npm-scripts-info": "^1.0.0", @@ -263,4 +264,4 @@ "**/*.spec.js" ] } -} \ No newline at end of file +} diff --git a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasExportApis.ts b/packages/nocodb/src/lib/meta/api/dataApis/dataAliasExportApis.ts index 7e18477427..5c393ced04 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasExportApis.ts +++ b/packages/nocodb/src/lib/meta/api/dataApis/dataAliasExportApis.ts @@ -1,12 +1,35 @@ import { Request, Response, Router } from 'express'; +import * as XLSX from 'xlsx'; import ncMetaAclMw from '../../helpers/ncMetaAclMw'; import { extractCsvData, + extractXlsxData, getViewAndModelFromRequestByAliasOrId, } from './helpers'; import apiMetrics from '../../helpers/apiMetrics'; import View from '../../../models/View'; +async function excelDataExport(req: Request, res: Response) { + const {model, view} = await getViewAndModelFromRequestByAliasOrId(req); + let targetView = view; + if (!targetView) { + targetView = await View.getDefaultView(model.id); + } + const { offset, elapsed, data } = await extractXlsxData(targetView, req); + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, data, targetView.title); + const buf = XLSX.write(wb, { type: "base64", bookType: "xlsx" }); + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + targetView.title + )}-export.xlsx"`, + }); + res.end(buf); +} + async function csvDataExport(req: Request, res: Response) { const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); let targetView = view; @@ -38,5 +61,15 @@ router.get( apiMetrics, ncMetaAclMw(csvDataExport, 'exportCsv') ); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/export/excel', + apiMetrics, + ncMetaAclMw(excelDataExport, 'exportExcel') +); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/excel', + apiMetrics, + ncMetaAclMw(excelDataExport, 'exportExcel') +); export default router; diff --git a/packages/nocodb/src/lib/meta/api/dataApis/helpers.ts b/packages/nocodb/src/lib/meta/api/dataApis/helpers.ts index 27d93d356e..19952ee9a1 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/helpers.ts +++ b/packages/nocodb/src/lib/meta/api/dataApis/helpers.ts @@ -8,6 +8,7 @@ import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; import { isSystemColumn, UITypes } from 'nocodb-sdk'; import { nocoExecute } from 'nc-help'; +import * as XLSX from 'xlsx'; import Column from '../../../models/Column'; import LookupColumn from '../../../models/LookupColumn'; import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; @@ -36,6 +37,75 @@ export async function getViewAndModelFromRequestByAliasOrId( return { model, view }; } +export async function extractXlsxData(view: View, req: Request) { + const base = await Base.get(view.base_id); + + await view.getModelWithInfo(); + await view.getColumns(); + + view.model.columns = view.columns + .filter((c) => c.show) + .map( + (c) => + new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any) + ) + .filter((column) => !isSystemColumn(column) || view.show_system_fields); + + const baseModel = await Model.getBaseModelSQL({ + id: view.model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + let offset = +req.query.offset || 0; + const limit = 100; + // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; + const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; + const csvRows = []; + const startTime = process.hrtime(); + let elapsed, temp; + + for ( + elapsed = 0; + elapsed < timeout; + offset += limit, + temp = process.hrtime(startTime), + elapsed = temp[0] * 1000 + temp[1] / 1000000 + ) { + const rows = await nocoExecute( + await getAst({ + query: req.query, + includePkByDefault: false, + model: view.model, + view, + }), + await baseModel.list({ ...req.query, offset, limit }), + {}, + req.query + ); + + if (!rows?.length) { + offset = -1; + break; + } + + for (const row of rows) { + const csvRow = { ...row }; + + for (const column of view.model.columns) { + if (isSystemColumn(column) && !view.show_system_fields) continue; + csvRow[column.title] = await serializeCellValue({ + value: row[column.title], + column, + }); + } + csvRows.push(csvRow); + } + } + const data = XLSX.utils.json_to_sheet(csvRows); + return { offset, csvRows, elapsed, data }; +} + export async function extractCsvData(view: View, req: Request) { const base = await Base.get(view.base_id); diff --git a/packages/nocodb/src/lib/utils/projectAcl.ts b/packages/nocodb/src/lib/utils/projectAcl.ts index ceb9fcd034..a3482a7137 100644 --- a/packages/nocodb/src/lib/utils/projectAcl.ts +++ b/packages/nocodb/src/lib/utils/projectAcl.ts @@ -27,6 +27,7 @@ export default { dataGroupBy: true, commentsCount: true, exportCsv: true, + exportExcel: true, viewList: true, columnList: true, @@ -142,6 +143,7 @@ export default { // project projectGet: true, exportCsv: true, + exportExcel: true, //table tableGet: true, @@ -205,6 +207,7 @@ export default { dataGroupBy: true, commentsCount: true, exportCsv: true, + exportExcel: true, // sort & filter sortList: true,