Browse Source

refactor: audit, cache handler and data export api(WIP)

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/5239/head
Pranav C 2 years ago
parent
commit
d4d8e8a9cd
  1. 22
      packages/nocodb/src/lib/controllers/auditController.ts
  2. 6
      packages/nocodb/src/lib/controllers/cacheController.ts
  3. 3
      packages/nocodb/src/lib/controllers/dataApis/helpers.ts
  4. 75
      packages/nocodb/src/lib/controllers/dataController/export.ts
  5. 2
      packages/nocodb/src/lib/controllers/tableController.ts
  6. 67
      packages/nocodb/src/lib/services/auditService.ts
  7. 10
      packages/nocodb/src/lib/services/cacheService.ts
  8. 54
      packages/nocodb/src/lib/services/dataService/export.ts
  9. 294
      packages/nocodb/src/lib/services/dataService/helpers.ts
  10. 2
      packages/nocodb/src/lib/services/dataService/index.ts
  11. 2
      packages/nocodb/src/lib/services/index.ts

22
packages/nocodb/src/lib/controllers/auditController.ts

@ -7,33 +7,25 @@ import ncMetaAclMw from '../meta/helpers/ncMetaAclMw';
import DOMPurify from 'isomorphic-dompurify'; import DOMPurify from 'isomorphic-dompurify';
import { getAjvValidatorMw } from '../meta/api/helpers'; import { getAjvValidatorMw } from '../meta/api/helpers';
import { auditService } from '../services';
export async function commentRow(req: Request<any, any>, res) { export async function commentRow(req: Request<any, any>, res) {
res.json( res.json(
await Audit.insert({ await Audit.insert({
...req.body, ...req.body,
user: (req as any).user?.email, user: (req as any).user,
op_type: AuditOperationTypes.COMMENT, op_type: AuditOperationTypes.COMMENT,
}) })
); );
} }
export async function auditRowUpdate(req: Request<any, any>, res) { export async function auditRowUpdate(req: Request<any, any>, res) {
const model = await Model.getByIdOrName({ id: req.body.fk_model_id });
res.json( res.json(
await Audit.insert({ await auditService.auditRowUpdate({
fk_model_id: req.body.fk_model_id, rowId: req.params.rowId,
row_id: req.params.rowId, body: {
op_type: AuditOperationTypes.DATA, ...req.body,
op_sub_type: AuditOperationSubTypes.UPDATE, },
description: DOMPurify.sanitize(
`Table ${model.table_name} : field ${req.body.column_name} got changed from ${req.body.prev_value} to ${req.body.value}`
),
details: DOMPurify.sanitize(`<span class="">${req.body.column_name}</span>
: <span class="text-decoration-line-through red px-2 lighten-4 black--text">${req.body.prev_value}</span>
<span class="black--text green lighten-4 px-2">${req.body.value}</span>`),
ip: (req as any).clientIp,
user: (req as any).user?.email,
}) })
); );
} }

6
packages/nocodb/src/lib/controllers/cacheController.ts

@ -1,9 +1,9 @@
import catchError from '../meta/helpers/catchError'; import catchError from '../meta/helpers/catchError';
import NocoCache from '../cache/NocoCache';
import { Router } from 'express'; import { Router } from 'express';
import { cacheService } from '../services';
export async function cacheGet(_, res) { export async function cacheGet(_, res) {
const data = await NocoCache.export(); const data = await cacheService.cacheGet();
res.set({ res.set({
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Content-Disposition': `attachment; filename="cache-export.json"`, 'Content-Disposition': `attachment; filename="cache-export.json"`,
@ -12,7 +12,7 @@ export async function cacheGet(_, res) {
} }
export async function cacheDelete(_, res) { export async function cacheDelete(_, res) {
return res.json(await NocoCache.destroy()); return res.json(await cacheService.cacheDelete());
} }
const router = Router(); const router = Router();

3
packages/nocodb/src/lib/controllers/dataApis/helpers.ts

@ -36,7 +36,8 @@ export async function getViewAndModelFromRequestByAliasOrId(
return { model, view }; return { model, view };
} }
export async function extractXlsxData(view: View, req: Request) { export async function extractXlsxData(param: {view: View, query:any}) {
const { view, query } = param;
const base = await Base.get(view.base_id); const base = await Base.get(view.base_id);
await view.getModelWithInfo(); await view.getModelWithInfo();

75
packages/nocodb/src/lib/controllers/dataController/export.ts

@ -0,0 +1,75 @@
import { Request, Response, Router } from 'express';
import * as XLSX from 'xlsx';
import ncMetaAclMw from '../../helpers/ncMetaAclMw';
import {
extractCsvData,
extractXlsxData,
getViewAndModelFromRequestByAliasOrId,
} from './helpers';
import apiMetrics from '../../helpers/apiMetrics';
import View from '../../../models/View';
async function excelDataExport(req: Request, res: Response) {
const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
let targetView = view;
if (!targetView) {
targetView = await View.getDefaultView(model.id);
}
const { offset, elapsed, data } = await extractXlsxData(targetView, req);
const wb = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(wb, data, targetView.title);
const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' });
res.set({
'Access-Control-Expose-Headers': 'nc-export-offset',
'nc-export-offset': offset,
'nc-export-elapsed-time': elapsed,
'Content-Disposition': `attachment; filename="${encodeURI(
targetView.title
)}-export.xlsx"`,
});
res.end(buf);
}
async function csvDataExport(req: Request, res: Response) {
const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
let targetView = view;
if (!targetView) {
targetView = await View.getDefaultView(model.id);
}
const { offset, elapsed, data } = await extractCsvData(targetView, req);
res.set({
'Access-Control-Expose-Headers': 'nc-export-offset',
'nc-export-offset': offset,
'nc-export-elapsed-time': elapsed,
'Content-Disposition': `attachment; filename="${encodeURI(
targetView.title
)}-export.csv"`,
});
res.send(data);
}
const router = Router({ mergeParams: true });
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/export/csv',
apiMetrics,
ncMetaAclMw(csvDataExport, 'exportCsv')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/csv',
apiMetrics,
ncMetaAclMw(csvDataExport, 'exportCsv')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/export/excel',
apiMetrics,
ncMetaAclMw(excelDataExport, 'exportExcel')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/excel',
apiMetrics,
ncMetaAclMw(excelDataExport, 'exportExcel')
);
export default router;

2
packages/nocodb/src/lib/controllers/tableController.ts

@ -22,7 +22,7 @@ export async function tableList(req: Request, res: Response<TableListType>) {
} }
export async function tableCreate(req: Request<any, any, TableReqType>, res) { export async function tableCreate(req: Request<any, any, TableReqType>, res) {
const result = tableService.createTable({ const result = tableService.tableCreate({
projectId: req.params.projectId, projectId: req.params.projectId,
baseId: req.params.baseId, baseId: req.params.baseId,
table: req.body, table: req.body,

67
packages/nocodb/src/lib/services/auditService.ts

@ -0,0 +1,67 @@
import {
AuditRowUpdatePayloadType,
CommentRowPayloadType,
} from 'nocodb-sdk/build/main/lib/CustomAPI';
import Audit from '../models/Audit';
import { AuditOperationSubTypes, AuditOperationTypes } from 'nocodb-sdk';
import Model from '../models/Model';
import { PagedResponseImpl } from '../meta/helpers/PagedResponse';
import DOMPurify from 'isomorphic-dompurify';
export async function commentRow(param: {
rowId: string;
body: CommentRowPayloadType;
user: any;
}) {
return await Audit.insert({
...param.body,
user: param.user?.email,
op_type: AuditOperationTypes.COMMENT,
});
}
export async function auditRowUpdate(param: {
rowId: string;
body: AuditRowUpdatePayloadType;
}) {
const model = await Model.getByIdOrName({ id: param.body.fk_model_id });
return await Audit.insert({
fk_model_id: param.body.fk_model_id,
row_id: param.rowId,
op_type: AuditOperationTypes.DATA,
op_sub_type: AuditOperationSubTypes.UPDATE,
description: DOMPurify.sanitize(
`Table ${model.table_name} : field ${param.body.column_name} got changed from ${param.body.prev_value} to ${param.body.value}`
),
details: DOMPurify.sanitize(`<span class="">${param.body.column_name}</span>
: <span class="text-decoration-line-through red px-2 lighten-4 black--text">${param.body.prev_value}</span>
<span class="black--text green lighten-4 px-2">${param.body.value}</span>`),
ip: (param as any).clientIp,
user: (param as any).user?.email,
});
}
export async function commentList(param: { query: any }) {
return await Audit.commentsList(param.query);
}
export async function auditList(param: { query: any; projectId: string }) {
return new PagedResponseImpl(
await Audit.projectAuditList(param.projectId, param.query),
{
count: await Audit.projectAuditCount(param.projectId),
...param.query,
}
);
}
export async function commentsCount(param: {
fk_model_id: string;
ids: string[];
}) {
return await Audit.commentsCount({
fk_model_id: param.fk_model_id as string,
ids: param.ids as string[],
});
}

10
packages/nocodb/src/lib/services/cacheService.ts

@ -0,0 +1,10 @@
import NocoCache from '../cache/NocoCache';
export async function cacheGet() {
return await NocoCache.export();
}
export async function cacheDelete() {
await NocoCache.destroy()
return true
}

54
packages/nocodb/src/lib/services/dataService/export.ts

@ -0,0 +1,54 @@
import { Request, Response, Router } from 'express';
import { isSystemColumn } from 'nocodb-sdk'
import * as XLSX from 'xlsx';
import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'
import ncMetaAclMw from '../../helpers/ncMetaAclMw';
import { extractXlsxData, serializeCellValue } from '../../meta/api/dataApis/helpers'
import {
extractCsvData, getViewAndModelByAliasOrId,
getViewAndModelFromRequestByAliasOrId, PathParams,
} from './helpers'
import apiMetrics from '../../helpers/apiMetrics';
import View from '../../../models/View';
async function excelDataExport(param:PathParams&{
query: any;
}) {
const { model, view } = await getViewAndModelByAliasOrId(param);
let targetView = view;
if (!targetView) {
targetView = await View.getDefaultView(model.id);
}
const { offset, elapsed, data } = await extractXlsxData({view: targetView, query:req.query });
const wb = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(wb, data, targetView.title);
const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' });
res.set({
'Access-Control-Expose-Headers': 'nc-export-offset',
'nc-export-offset': offset,
'nc-export-elapsed-time': elapsed,
'Content-Disposition': `attachment; filename="${encodeURI(
targetView.title
)}-export.xlsx"`,
});
res.end(buf);
}
async function csvDataExport(req: Request, res: Response) {
const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
let targetView = view;
if (!targetView) {
targetView = await View.getDefaultView(model.id);
}
const { offset, elapsed, data } = await extractCsvData(targetView, req);
res.set({
'Access-Control-Expose-Headers': 'nc-export-offset',
'nc-export-offset': offset,
'nc-export-elapsed-time': elapsed,
'Content-Disposition': `attachment; filename="${encodeURI(
targetView.title
)}-export.csv"`,
});
res.send(data);
}

294
packages/nocodb/src/lib/services/dataService/helpers.ts

@ -1,6 +1,17 @@
import { Request } from 'express'
import { nocoExecute } from 'nc-help'
import { isSystemColumn, UITypes } from 'nocodb-sdk'
import * as XLSX from 'xlsx'
import { BaseModelSqlv2 } from '../../db/sql-data-mapper/lib/sql/BaseModelSqlv2'
import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'
import { NcError } from '../../meta/helpers/catchError' import { NcError } from '../../meta/helpers/catchError'
import { Model, View } from '../../models' import { Model, View } from '../../models'
import Base from '../../models/Base'
import Column from '../../models/Column'
import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'
import LookupColumn from '../../models/LookupColumn'
import Project from '../../models/Project' import Project from '../../models/Project'
import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'
export interface PathParams { export interface PathParams {
@ -29,3 +40,286 @@ export async function getViewAndModelByAliasOrId(param: {
if (!model) NcError.notFound('Table not found'); if (!model) NcError.notFound('Table not found');
return { model, view }; return { model, view };
} }
export async function extractXlsxData(view: View, req: Request) {
const base = await Base.get(view.base_id);
await view.getModelWithInfo();
await view.getColumns();
view.model.columns = view.columns
.filter((c) => c.show)
.map(
(c) =>
new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any)
)
.filter((column) => !isSystemColumn(column) || view.show_system_fields);
const baseModel = await Model.getBaseModelSQL({
id: view.model.id,
viewId: view?.id,
dbDriver: NcConnectionMgrv2.get(base),
});
const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req);
const fields = req.query.fields as string[];
const data = XLSX.utils.json_to_sheet(dbRows, { header: fields });
return { offset, dbRows, elapsed, data };
}
export async function extractCsvData(view: View, req: Request) {
const base = await Base.get(view.base_id);
const fields = req.query.fields;
await view.getModelWithInfo();
await view.getColumns();
view.model.columns = view.columns
.filter((c) => c.show)
.map(
(c) =>
new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any)
)
.filter((column) => !isSystemColumn(column) || view.show_system_fields);
const baseModel = await Model.getBaseModelSQL({
id: view.model.id,
viewId: view?.id,
dbDriver: NcConnectionMgrv2.get(base),
});
const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req);
const data = papaparse.unparse(
{
fields: view.model.columns
.sort((c1, c2) =>
Array.isArray(fields)
? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any)
: 0
)
.filter(
(c) =>
!fields || !Array.isArray(fields) || fields.includes(c.title as any)
)
.map((c) => c.title),
data: dbRows,
},
{
escapeFormulae: true,
}
);
return { offset, dbRows, elapsed, data };
}
async function getDbRows(baseModel, view: View, req: Request) {
let offset = +req.query.offset || 0;
const limit = 100;
// const size = +process.env.NC_EXPORT_MAX_SIZE || 1024;
const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000;
const dbRows = [];
const startTime = process.hrtime();
let elapsed, temp;
const listArgs: any = { ...req.query };
try {
listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
} catch (e) {}
try {
listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
} catch (e) {}
for (
elapsed = 0;
elapsed < timeout;
offset += limit,
temp = process.hrtime(startTime),
elapsed = temp[0] * 1000 + temp[1] / 1000000
) {
const rows = await nocoExecute(
await getAst({
query: req.query,
includePkByDefault: false,
model: view.model,
view,
}),
await baseModel.list({ ...listArgs, offset, limit }),
{},
req.query
);
if (!rows?.length) {
offset = -1;
break;
}
for (const row of rows) {
const dbRow = { ...row };
for (const column of view.model.columns) {
if (isSystemColumn(column) && !view.show_system_fields) continue;
dbRow[column.title] = await serializeCellValue({
value: row[column.title],
column,
siteUrl: req['ncSiteUrl'],
});
}
dbRows.push(dbRow);
}
}
return { offset, dbRows, elapsed };
}
export async function serializeCellValue({
value,
column,
siteUrl,
}: {
column?: Column;
value: any;
siteUrl: string;
}) {
if (!column) {
return value;
}
if (!value) return value;
switch (column?.uidt) {
case UITypes.Attachment: {
let data = value;
try {
if (typeof value === 'string') {
data = JSON.parse(value);
}
} catch {}
return (data || []).map(
(attachment) =>
`${encodeURI(attachment.title)}(${encodeURI(
attachment.path ? `${siteUrl}/${attachment.path}` : attachment.url
)})`
);
}
case UITypes.Lookup:
{
const colOptions = await column.getColOptions<LookupColumn>();
const lookupColumn = await colOptions.getLookupColumn();
return (
await Promise.all(
[...(Array.isArray(value) ? value : [value])].map(async (v) =>
serializeCellValue({
value: v,
column: lookupColumn,
siteUrl,
})
)
)
).join(', ');
}
break;
case UITypes.LinkToAnotherRecord:
{
const colOptions =
await column.getColOptions<LinkToAnotherRecordColumn>();
const relatedModel = await colOptions.getRelatedTable();
await relatedModel.getColumns();
return [...(Array.isArray(value) ? value : [value])]
.map((v) => {
return v[relatedModel.displayValue?.title];
})
.join(', ');
}
break;
default:
if (value && typeof value === 'object') {
return JSON.stringify(value);
}
return value;
}
}
export async function getColumnByIdOrName(
columnNameOrId: string,
model: Model
) {
const column = (await model.getColumns()).find(
(c) =>
c.title === columnNameOrId ||
c.id === columnNameOrId ||
c.column_name === columnNameOrId
);
if (!column)
NcError.notFound(`Column with id/name '${columnNameOrId}' is not found`);
return column;
}
async function getDbRows(param: { baseModel:BaseModelSqlv2, view: View, query: any; siteUrl: string; }) {
const { baseModel, view, query = {}, siteUrl } = param;
let offset = +query.offset || 0;
const limit = 100;
// const size = +process.env.NC_EXPORT_MAX_SIZE || 1024;
const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000;
const dbRows = [];
const startTime = process.hrtime();
let elapsed, temp;
const listArgs: any = { ...query };
try {
listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
} catch (e) {}
try {
listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
} catch (e) {}
for (
elapsed = 0;
elapsed < timeout;
offset += limit,
temp = process.hrtime(startTime),
elapsed = temp[0] * 1000 + temp[1] / 1000000
) {
const rows = await nocoExecute(
await getAst({
query: query,
includePkByDefault: false,
model: view.model,
view,
}),
await baseModel.list({ ...listArgs, offset, limit }),
{},
query
);
if (!rows?.length) {
offset = -1;
break;
}
for (const row of rows) {
const dbRow = { ...row };
for (const column of view.model.columns) {
if (isSystemColumn(column) && !view.show_system_fields) continue;
dbRow[column.title] = await serializeCellValue({
value: row[column.title],
column,
siteUrl,
});
}
dbRows.push(dbRow);
}
}
return { offset, dbRows, elapsed };
}

2
packages/nocodb/src/lib/services/dataService/index.ts

@ -1,4 +1,4 @@
import { nocoExecute } from 'nc-help/dist/module/NocoExecute'; import { nocoExecute } from 'nc-help';
import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst';
import { NcError } from '../../meta/helpers/catchError'; import { NcError } from '../../meta/helpers/catchError';
import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; import { PagedResponseImpl } from '../../meta/helpers/PagedResponse';

2
packages/nocodb/src/lib/services/index.ts

@ -28,3 +28,5 @@ export * as attachmentService from './attachmentService';
export * as hookFilterService from './hookFilterService'; export * as hookFilterService from './hookFilterService';
export * as dataService from './dataService'; export * as dataService from './dataService';
export * as bulkDataService from './dataService/bulkData'; export * as bulkDataService from './dataService/bulkData';
export * as cacheService from './cacheService';
export * as auditService from './auditService';

Loading…
Cancel
Save