From d4d8e8a9cd9b2b1a5a8f8a70b048c57a06434975 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 00:23:01 +0530 Subject: [PATCH] refactor: audit, cache handler and data export api(WIP) Signed-off-by: Pranav C --- .../src/lib/controllers/auditController.ts | 22 +- .../src/lib/controllers/cacheController.ts | 6 +- .../src/lib/controllers/dataApis/helpers.ts | 3 +- .../lib/controllers/dataController/export.ts | 75 +++++ .../src/lib/controllers/tableController.ts | 2 +- .../nocodb/src/lib/services/auditService.ts | 67 ++++ .../nocodb/src/lib/services/cacheService.ts | 10 + .../src/lib/services/dataService/export.ts | 54 ++++ .../src/lib/services/dataService/helpers.ts | 294 ++++++++++++++++++ .../src/lib/services/dataService/index.ts | 2 +- packages/nocodb/src/lib/services/index.ts | 2 + 11 files changed, 516 insertions(+), 21 deletions(-) create mode 100644 packages/nocodb/src/lib/controllers/dataController/export.ts create mode 100644 packages/nocodb/src/lib/services/auditService.ts create mode 100644 packages/nocodb/src/lib/services/cacheService.ts create mode 100644 packages/nocodb/src/lib/services/dataService/export.ts diff --git a/packages/nocodb/src/lib/controllers/auditController.ts b/packages/nocodb/src/lib/controllers/auditController.ts index 96b0acddee..b5d473cb60 100644 --- a/packages/nocodb/src/lib/controllers/auditController.ts +++ b/packages/nocodb/src/lib/controllers/auditController.ts @@ -7,33 +7,25 @@ import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import DOMPurify from 'isomorphic-dompurify'; import { getAjvValidatorMw } from '../meta/api/helpers'; +import { auditService } from '../services'; export async function commentRow(req: Request, res) { res.json( await Audit.insert({ ...req.body, - user: (req as any).user?.email, + user: (req as any).user, op_type: AuditOperationTypes.COMMENT, }) ); } export async function auditRowUpdate(req: Request, res) { - const model = await Model.getByIdOrName({ id: req.body.fk_model_id }); res.json( - await Audit.insert({ - fk_model_id: req.body.fk_model_id, - row_id: req.params.rowId, - op_type: AuditOperationTypes.DATA, - op_sub_type: AuditOperationSubTypes.UPDATE, - description: DOMPurify.sanitize( - `Table ${model.table_name} : field ${req.body.column_name} got changed from ${req.body.prev_value} to ${req.body.value}` - ), - details: DOMPurify.sanitize(`${req.body.column_name} - : ${req.body.prev_value} - ${req.body.value}`), - ip: (req as any).clientIp, - user: (req as any).user?.email, + await auditService.auditRowUpdate({ + rowId: req.params.rowId, + body: { + ...req.body, + }, }) ); } diff --git a/packages/nocodb/src/lib/controllers/cacheController.ts b/packages/nocodb/src/lib/controllers/cacheController.ts index 9e1fab6777..751c26178c 100644 --- a/packages/nocodb/src/lib/controllers/cacheController.ts +++ b/packages/nocodb/src/lib/controllers/cacheController.ts @@ -1,9 +1,9 @@ import catchError from '../meta/helpers/catchError'; -import NocoCache from '../cache/NocoCache'; import { Router } from 'express'; +import { cacheService } from '../services'; export async function cacheGet(_, res) { - const data = await NocoCache.export(); + const data = await cacheService.cacheGet(); res.set({ 'Content-Type': 'application/json', 'Content-Disposition': `attachment; filename="cache-export.json"`, @@ -12,7 +12,7 @@ export async function cacheGet(_, res) { } export async function cacheDelete(_, res) { - return res.json(await NocoCache.destroy()); + return res.json(await cacheService.cacheDelete()); } const router = Router(); diff --git a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts index a2b94737f2..ed48507e0d 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts @@ -36,7 +36,8 @@ export async function getViewAndModelFromRequestByAliasOrId( return { model, view }; } -export async function extractXlsxData(view: View, req: Request) { +export async function extractXlsxData(param: {view: View, query:any}) { + const { view, query } = param; const base = await Base.get(view.base_id); await view.getModelWithInfo(); diff --git a/packages/nocodb/src/lib/controllers/dataController/export.ts b/packages/nocodb/src/lib/controllers/dataController/export.ts new file mode 100644 index 0000000000..8c6cf38bea --- /dev/null +++ b/packages/nocodb/src/lib/controllers/dataController/export.ts @@ -0,0 +1,75 @@ +import { Request, Response, Router } from 'express'; +import * as XLSX from 'xlsx'; +import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import { + extractCsvData, + extractXlsxData, + getViewAndModelFromRequestByAliasOrId, +} from './helpers'; +import apiMetrics from '../../helpers/apiMetrics'; +import View from '../../../models/View'; + +async function excelDataExport(req: Request, res: Response) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + let targetView = view; + if (!targetView) { + targetView = await View.getDefaultView(model.id); + } + const { offset, elapsed, data } = await extractXlsxData(targetView, req); + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, data, targetView.title); + const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + targetView.title + )}-export.xlsx"`, + }); + res.end(buf); +} + +async function csvDataExport(req: Request, res: Response) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + let targetView = view; + if (!targetView) { + targetView = await View.getDefaultView(model.id); + } + const { offset, elapsed, data } = await extractCsvData(targetView, req); + + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + targetView.title + )}-export.csv"`, + }); + res.send(data); +} + +const router = Router({ mergeParams: true }); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/export/csv', + apiMetrics, + ncMetaAclMw(csvDataExport, 'exportCsv') +); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/csv', + apiMetrics, + ncMetaAclMw(csvDataExport, 'exportCsv') +); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/export/excel', + apiMetrics, + ncMetaAclMw(excelDataExport, 'exportExcel') +); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/excel', + apiMetrics, + ncMetaAclMw(excelDataExport, 'exportExcel') +); + +export default router; diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index 728cc1816a..8ee37c019f 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -22,7 +22,7 @@ export async function tableList(req: Request, res: Response) { } export async function tableCreate(req: Request, res) { - const result = tableService.createTable({ + const result = tableService.tableCreate({ projectId: req.params.projectId, baseId: req.params.baseId, table: req.body, diff --git a/packages/nocodb/src/lib/services/auditService.ts b/packages/nocodb/src/lib/services/auditService.ts new file mode 100644 index 0000000000..211c8d4f2d --- /dev/null +++ b/packages/nocodb/src/lib/services/auditService.ts @@ -0,0 +1,67 @@ +import { + AuditRowUpdatePayloadType, + CommentRowPayloadType, +} from 'nocodb-sdk/build/main/lib/CustomAPI'; +import Audit from '../models/Audit'; +import { AuditOperationSubTypes, AuditOperationTypes } from 'nocodb-sdk'; +import Model from '../models/Model'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; + +import DOMPurify from 'isomorphic-dompurify'; + +export async function commentRow(param: { + rowId: string; + body: CommentRowPayloadType; + user: any; +}) { + return await Audit.insert({ + ...param.body, + user: param.user?.email, + op_type: AuditOperationTypes.COMMENT, + }); +} + +export async function auditRowUpdate(param: { + rowId: string; + body: AuditRowUpdatePayloadType; +}) { + const model = await Model.getByIdOrName({ id: param.body.fk_model_id }); + return await Audit.insert({ + fk_model_id: param.body.fk_model_id, + row_id: param.rowId, + op_type: AuditOperationTypes.DATA, + op_sub_type: AuditOperationSubTypes.UPDATE, + description: DOMPurify.sanitize( + `Table ${model.table_name} : field ${param.body.column_name} got changed from ${param.body.prev_value} to ${param.body.value}` + ), + details: DOMPurify.sanitize(`${param.body.column_name} + : ${param.body.prev_value} + ${param.body.value}`), + ip: (param as any).clientIp, + user: (param as any).user?.email, + }); +} + +export async function commentList(param: { query: any }) { + return await Audit.commentsList(param.query); +} + +export async function auditList(param: { query: any; projectId: string }) { + return new PagedResponseImpl( + await Audit.projectAuditList(param.projectId, param.query), + { + count: await Audit.projectAuditCount(param.projectId), + ...param.query, + } + ); +} + +export async function commentsCount(param: { + fk_model_id: string; + ids: string[]; +}) { + return await Audit.commentsCount({ + fk_model_id: param.fk_model_id as string, + ids: param.ids as string[], + }); +} diff --git a/packages/nocodb/src/lib/services/cacheService.ts b/packages/nocodb/src/lib/services/cacheService.ts new file mode 100644 index 0000000000..6e3b83df86 --- /dev/null +++ b/packages/nocodb/src/lib/services/cacheService.ts @@ -0,0 +1,10 @@ +import NocoCache from '../cache/NocoCache'; + +export async function cacheGet() { + return await NocoCache.export(); +} + +export async function cacheDelete() { + await NocoCache.destroy() + return true +} diff --git a/packages/nocodb/src/lib/services/dataService/export.ts b/packages/nocodb/src/lib/services/dataService/export.ts new file mode 100644 index 0000000000..2ebc209262 --- /dev/null +++ b/packages/nocodb/src/lib/services/dataService/export.ts @@ -0,0 +1,54 @@ +import { Request, Response, Router } from 'express'; +import { isSystemColumn } from 'nocodb-sdk' +import * as XLSX from 'xlsx'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst' +import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import { extractXlsxData, serializeCellValue } from '../../meta/api/dataApis/helpers' +import { + extractCsvData, getViewAndModelByAliasOrId, + getViewAndModelFromRequestByAliasOrId, PathParams, +} from './helpers' +import apiMetrics from '../../helpers/apiMetrics'; +import View from '../../../models/View'; + +async function excelDataExport(param:PathParams&{ + query: any; +}) { + const { model, view } = await getViewAndModelByAliasOrId(param); + let targetView = view; + if (!targetView) { + targetView = await View.getDefaultView(model.id); + } + const { offset, elapsed, data } = await extractXlsxData({view: targetView, query:req.query }); + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, data, targetView.title); + const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + targetView.title + )}-export.xlsx"`, + }); + res.end(buf); +} + +async function csvDataExport(req: Request, res: Response) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + let targetView = view; + if (!targetView) { + targetView = await View.getDefaultView(model.id); + } + const { offset, elapsed, data } = await extractCsvData(targetView, req); + + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + targetView.title + )}-export.csv"`, + }); + res.send(data); +} diff --git a/packages/nocodb/src/lib/services/dataService/helpers.ts b/packages/nocodb/src/lib/services/dataService/helpers.ts index bed4f78f3a..02c56c7a39 100644 --- a/packages/nocodb/src/lib/services/dataService/helpers.ts +++ b/packages/nocodb/src/lib/services/dataService/helpers.ts @@ -1,6 +1,17 @@ +import { Request } from 'express' +import { nocoExecute } from 'nc-help' +import { isSystemColumn, UITypes } from 'nocodb-sdk' +import * as XLSX from 'xlsx' +import { BaseModelSqlv2 } from '../../db/sql-data-mapper/lib/sql/BaseModelSqlv2' +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst' import { NcError } from '../../meta/helpers/catchError' import { Model, View } from '../../models' +import Base from '../../models/Base' +import Column from '../../models/Column' +import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn' +import LookupColumn from '../../models/LookupColumn' import Project from '../../models/Project' +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2' export interface PathParams { @@ -29,3 +40,286 @@ export async function getViewAndModelByAliasOrId(param: { if (!model) NcError.notFound('Table not found'); return { model, view }; } + + + +export async function extractXlsxData(view: View, req: Request) { + const base = await Base.get(view.base_id); + + await view.getModelWithInfo(); + await view.getColumns(); + + view.model.columns = view.columns + .filter((c) => c.show) + .map( + (c) => + new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any) + ) + .filter((column) => !isSystemColumn(column) || view.show_system_fields); + + const baseModel = await Model.getBaseModelSQL({ + id: view.model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req); + + const fields = req.query.fields as string[]; + + const data = XLSX.utils.json_to_sheet(dbRows, { header: fields }); + + return { offset, dbRows, elapsed, data }; +} + +export async function extractCsvData(view: View, req: Request) { + const base = await Base.get(view.base_id); + const fields = req.query.fields; + + await view.getModelWithInfo(); + await view.getColumns(); + + view.model.columns = view.columns + .filter((c) => c.show) + .map( + (c) => + new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any) + ) + .filter((column) => !isSystemColumn(column) || view.show_system_fields); + + const baseModel = await Model.getBaseModelSQL({ + id: view.model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req); + + const data = papaparse.unparse( + { + fields: view.model.columns + .sort((c1, c2) => + Array.isArray(fields) + ? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any) + : 0 + ) + .filter( + (c) => + !fields || !Array.isArray(fields) || fields.includes(c.title as any) + ) + .map((c) => c.title), + data: dbRows, + }, + { + escapeFormulae: true, + } + ); + + return { offset, dbRows, elapsed, data }; +} + + +async function getDbRows(baseModel, view: View, req: Request) { + let offset = +req.query.offset || 0; + const limit = 100; + // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; + const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; + const dbRows = []; + const startTime = process.hrtime(); + let elapsed, temp; + + const listArgs: any = { ...req.query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + + for ( + elapsed = 0; + elapsed < timeout; + offset += limit, + temp = process.hrtime(startTime), + elapsed = temp[0] * 1000 + temp[1] / 1000000 + ) { + const rows = await nocoExecute( + await getAst({ + query: req.query, + includePkByDefault: false, + model: view.model, + view, + }), + await baseModel.list({ ...listArgs, offset, limit }), + {}, + req.query + ); + + if (!rows?.length) { + offset = -1; + break; + } + + for (const row of rows) { + const dbRow = { ...row }; + + for (const column of view.model.columns) { + if (isSystemColumn(column) && !view.show_system_fields) continue; + dbRow[column.title] = await serializeCellValue({ + value: row[column.title], + column, + siteUrl: req['ncSiteUrl'], + }); + } + dbRows.push(dbRow); + } + } + return { offset, dbRows, elapsed }; +} + +export async function serializeCellValue({ + value, + column, + siteUrl, + }: { + column?: Column; + value: any; + siteUrl: string; +}) { + if (!column) { + return value; + } + + if (!value) return value; + + switch (column?.uidt) { + case UITypes.Attachment: { + let data = value; + try { + if (typeof value === 'string') { + data = JSON.parse(value); + } + } catch {} + + return (data || []).map( + (attachment) => + `${encodeURI(attachment.title)}(${encodeURI( + attachment.path ? `${siteUrl}/${attachment.path}` : attachment.url + )})` + ); + } + case UITypes.Lookup: + { + const colOptions = await column.getColOptions(); + const lookupColumn = await colOptions.getLookupColumn(); + return ( + await Promise.all( + [...(Array.isArray(value) ? value : [value])].map(async (v) => + serializeCellValue({ + value: v, + column: lookupColumn, + siteUrl, + }) + ) + ) + ).join(', '); + } + break; + case UITypes.LinkToAnotherRecord: + { + const colOptions = + await column.getColOptions(); + const relatedModel = await colOptions.getRelatedTable(); + await relatedModel.getColumns(); + return [...(Array.isArray(value) ? value : [value])] + .map((v) => { + return v[relatedModel.displayValue?.title]; + }) + .join(', '); + } + break; + default: + if (value && typeof value === 'object') { + return JSON.stringify(value); + } + return value; + } +} + +export async function getColumnByIdOrName( + columnNameOrId: string, + model: Model +) { + const column = (await model.getColumns()).find( + (c) => + c.title === columnNameOrId || + c.id === columnNameOrId || + c.column_name === columnNameOrId + ); + + if (!column) + NcError.notFound(`Column with id/name '${columnNameOrId}' is not found`); + + return column; +} + + + +async function getDbRows(param: { baseModel:BaseModelSqlv2, view: View, query: any; siteUrl: string; }) { + const { baseModel, view, query = {}, siteUrl } = param; + let offset = +query.offset || 0; + const limit = 100; + // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; + const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; + const dbRows = []; + const startTime = process.hrtime(); + let elapsed, temp; + + const listArgs: any = { ...query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + + for ( + elapsed = 0; + elapsed < timeout; + offset += limit, + temp = process.hrtime(startTime), + elapsed = temp[0] * 1000 + temp[1] / 1000000 + ) { + const rows = await nocoExecute( + await getAst({ + query: query, + includePkByDefault: false, + model: view.model, + view, + }), + await baseModel.list({ ...listArgs, offset, limit }), + {}, + query + ); + + if (!rows?.length) { + offset = -1; + break; + } + + for (const row of rows) { + const dbRow = { ...row }; + + for (const column of view.model.columns) { + if (isSystemColumn(column) && !view.show_system_fields) continue; + dbRow[column.title] = await serializeCellValue({ + value: row[column.title], + column, + siteUrl, + }); + } + dbRows.push(dbRow); + } + } + return { offset, dbRows, elapsed }; +} diff --git a/packages/nocodb/src/lib/services/dataService/index.ts b/packages/nocodb/src/lib/services/dataService/index.ts index 0dd7ba3d02..668da2a598 100644 --- a/packages/nocodb/src/lib/services/dataService/index.ts +++ b/packages/nocodb/src/lib/services/dataService/index.ts @@ -1,4 +1,4 @@ -import { nocoExecute } from 'nc-help/dist/module/NocoExecute'; +import { nocoExecute } from 'nc-help'; import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; import { NcError } from '../../meta/helpers/catchError'; import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 4a5e9dacae..a9e46f8caa 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -28,3 +28,5 @@ export * as attachmentService from './attachmentService'; export * as hookFilterService from './hookFilterService'; export * as dataService from './dataService'; export * as bulkDataService from './dataService/bulkData'; +export * as cacheService from './cacheService'; +export * as auditService from './auditService';