github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: add data endpoint to export as excel file

pull/2851/head
Louis Delbosc 2 years ago
parent
5280d7b20d
commit
571651873f
6 changed files with 251 additions and 3 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      packages/nocodb-sdk/src/lib/globals.ts
  2. 142
      packages/nocodb/package-lock.json
  3. 5
      packages/nocodb/package.json
  4. 33
      packages/nocodb/src/lib/meta/api/dataApis/dataAliasExportApis.ts
  5. 70
      packages/nocodb/src/lib/meta/api/dataApis/helpers.ts
  6. 3
      packages/nocodb/src/lib/utils/projectAcl.ts

1
packages/nocodb-sdk/src/lib/globals.ts
Unescape View File

@ -12,6 +12,7 @@ export enum RelationTypes {
}
export enum ExportTypes {
EXCEL = 'excel',
CSV = 'csv',
}

142
packages/nocodb/package-lock.json generated
Unescape View File

@ -100,7 +100,8 @@
"unique-names-generator": "^4.3.1",
"uuid": "^8.2.0",
"validator": "^13.1.1",
"xc-core-ts": "^0.1.0"
"xc-core-ts": "^0.1.0",
"xlsx": "^0.18.5"
},
"devDependencies": {
"@bitjson/npm-scripts-info": "^1.0.0",
@ -2501,6 +2502,14 @@
"integrity": "sha512-qQLMr+8o0WC4FZGQTcJiKBVC59JylcPSrTtk6usvmIDFUOCKegapy1VHQwRbFMOFyb/inzUVqHs+eMYKDM1YeQ==",
"dev": true
},
"node_modules/adler-32": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
@ -4387,6 +4396,18 @@
"integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==",
"optional": true
},
"node_modules/cfb": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz",
"integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==",
"dependencies": {
"adler-32": "~1.3.0",
"crc-32": "~1.2.0"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/chai": {
"version": "4.3.6",
"resolved": "https://registry.npmjs.org/chai/-/chai-4.3.6.tgz",
@ -4926,6 +4947,14 @@
"node": ">=10"
}
},
"node_modules/codepage": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
"integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/collection-visit": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
@ -9887,6 +9916,14 @@
"node": ">= 0.6"
}
},
"node_modules/frac": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
"integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/fragment-cache": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
@ -19930,6 +19967,17 @@
"node": ">= 0.6"
}
},
"node_modules/ssf": {
"version": "0.11.2",
"resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
"integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
"dependencies": {
"frac": "~1.1.2"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/sshpk": {
"version": "1.17.0",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz",
@ -24426,6 +24474,22 @@
"node": ">= 6.4.0"
}
},
"node_modules/wmf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
"integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/word": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
"integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/word-wrap": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
@ -24582,6 +24646,26 @@
"node": ">=0.10.0"
}
},
"node_modules/xlsx": {
"version": "0.18.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
"integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
"dependencies": {
"adler-32": "~1.3.0",
"cfb": "~1.2.1",
"codepage": "~1.15.0",
"crc-32": "~1.2.1",
"ssf": "~0.11.2",
"wmf": "~1.0.1",
"word": "~0.3.0"
},
"bin": {
"xlsx": "bin/xlsx.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/xml": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz",
@ -26624,6 +26708,11 @@
"integrity": "sha512-qQLMr+8o0WC4FZGQTcJiKBVC59JylcPSrTtk6usvmIDFUOCKegapy1VHQwRbFMOFyb/inzUVqHs+eMYKDM1YeQ==",
"dev": true
},
"adler-32": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A=="
},
"agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
@ -28156,6 +28245,15 @@
"integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==",
"optional": true
},
"cfb": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz",
"integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==",
"requires": {
"adler-32": "~1.3.0",
"crc-32": "~1.2.0"
}
},
"chai": {
"version": "4.3.6",
"resolved": "https://registry.npmjs.org/chai/-/chai-4.3.6.tgz",
@ -28569,6 +28667,11 @@
}
}
},
"codepage": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
"integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA=="
},
"collection-visit": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
@ -32481,6 +32584,11 @@
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
"integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="
},
"frac": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
"integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA=="
},
"fragment-cache": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
@ -40312,6 +40420,14 @@
"resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz",
"integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg=="
},
"ssf": {
"version": "0.11.2",
"resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
"integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
"requires": {
"frac": "~1.1.2"
}
},
"sshpk": {
"version": "1.17.0",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz",
@ -43869,6 +43985,16 @@
"triple-beam": "^1.3.0"
}
},
"wmf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
"integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw=="
},
"word": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
"integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA=="
},
"word-wrap": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
@ -43985,6 +44111,20 @@
}
}
},
"xlsx": {
"version": "0.18.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
"integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
"requires": {
"adler-32": "~1.3.0",
"cfb": "~1.2.1",
"codepage": "~1.15.0",
"crc-32": "~1.2.1",
"ssf": "~0.11.2",
"wmf": "~1.0.1",
"word": "~0.3.0"
}
},
"xml": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz",

5
packages/nocodb/package.json
Unescape View File

@ -186,7 +186,8 @@
"unique-names-generator": "^4.3.1",
"uuid": "^8.2.0",
"validator": "^13.1.1",
"xc-core-ts": "^0.1.0"
"xc-core-ts": "^0.1.0",
"xlsx": "^0.18.5"
},
"devDependencies": {
"@bitjson/npm-scripts-info": "^1.0.0",
@ -263,4 +264,4 @@
"**/*.spec.js"
]
}
}
}

33
packages/nocodb/src/lib/meta/api/dataApis/dataAliasExportApis.ts
Unescape View File

@ -1,12 +1,35 @@
import { Request, Response, Router } from 'express';
import * as XLSX from 'xlsx';
import ncMetaAclMw from '../../helpers/ncMetaAclMw';
import {
extractCsvData,
extractXlsxData,
getViewAndModelFromRequestByAliasOrId,
} from './helpers';
import apiMetrics from '../../helpers/apiMetrics';
import View from '../../../models/View';
async function excelDataExport(req: Request, res: Response) {
const {model, view} = await getViewAndModelFromRequestByAliasOrId(req);
let targetView = view;
if (!targetView) {
targetView = await View.getDefaultView(model.id);
}
const { offset, elapsed, data } = await extractXlsxData(targetView, req);
const wb = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(wb, data, targetView.title);
const buf = XLSX.write(wb, { type: "base64", bookType: "xlsx" });
res.set({
'Access-Control-Expose-Headers': 'nc-export-offset',
'nc-export-offset': offset,
'nc-export-elapsed-time': elapsed,
'Content-Disposition': `attachment; filename="${encodeURI(
targetView.title
)}-export.xlsx"`,
});
res.end(buf);
}
async function csvDataExport(req: Request, res: Response) {
const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
let targetView = view;
@ -38,5 +61,15 @@ router.get(
apiMetrics,
ncMetaAclMw(csvDataExport, 'exportCsv')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/export/excel',
apiMetrics,
ncMetaAclMw(excelDataExport, 'exportExcel')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/excel',
apiMetrics,
ncMetaAclMw(excelDataExport, 'exportExcel')
);
export default router;

70
packages/nocodb/src/lib/meta/api/dataApis/helpers.ts
Unescape View File

@ -8,6 +8,7 @@ import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
import { isSystemColumn, UITypes } from 'nocodb-sdk';
import { nocoExecute } from 'nc-help';
import * as XLSX from 'xlsx';
import Column from '../../../models/Column';
import LookupColumn from '../../../models/LookupColumn';
import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn';
@ -36,6 +37,75 @@ export async function getViewAndModelFromRequestByAliasOrId(
return { model, view };
}
export async function extractXlsxData(view: View, req: Request) {
const base = await Base.get(view.base_id);
await view.getModelWithInfo();
await view.getColumns();
view.model.columns = view.columns
.filter((c) => c.show)
.map(
(c) =>
new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any)
)
.filter((column) => !isSystemColumn(column) || view.show_system_fields);
const baseModel = await Model.getBaseModelSQL({
id: view.model.id,
viewId: view?.id,
dbDriver: NcConnectionMgrv2.get(base),
});
let offset = +req.query.offset || 0;
const limit = 100;
// const size = +process.env.NC_EXPORT_MAX_SIZE || 1024;
const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000;
const csvRows = [];
const startTime = process.hrtime();
let elapsed, temp;
for (
elapsed = 0;
elapsed < timeout;
offset += limit,
temp = process.hrtime(startTime),
elapsed = temp[0] * 1000 + temp[1] / 1000000
) {
const rows = await nocoExecute(
await getAst({
query: req.query,
includePkByDefault: false,
model: view.model,
view,
}),
await baseModel.list({ ...req.query, offset, limit }),
{},
req.query
);
if (!rows?.length) {
offset = -1;
break;
}
for (const row of rows) {
const csvRow = { ...row };
for (const column of view.model.columns) {
if (isSystemColumn(column) && !view.show_system_fields) continue;
csvRow[column.title] = await serializeCellValue({
value: row[column.title],
column,
});
}
csvRows.push(csvRow);
}
}
const data = XLSX.utils.json_to_sheet(csvRows);
return { offset, csvRows, elapsed, data };
}
export async function extractCsvData(view: View, req: Request) {
const base = await Base.get(view.base_id);

3
packages/nocodb/src/lib/utils/projectAcl.ts
Unescape View File

@ -27,6 +27,7 @@ export default {
dataGroupBy: true,
commentsCount: true,
exportCsv: true,
exportExcel: true,
viewList: true,
columnList: true,
@ -142,6 +143,7 @@ export default {
// project
projectGet: true,
exportCsv: true,
exportExcel: true,
//table
tableGet: true,
@ -205,6 +207,7 @@ export default {
dataGroupBy: true,
commentsCount: true,
exportCsv: true,
exportExcel: true,
// sort & filter
sortList: true,

Write Preview
Loading…
Cancel
Save