refactor: audit, cache handler and data export api(WIP)

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb/src/lib/controllers/auditController.ts

@@ -7,33 +7,25 @@ import ncMetaAclMw from '../meta/helpers/ncMetaAclMw';
 
 import DOMPurify from 'isomorphic-dompurify';
 import { getAjvValidatorMw } from '../meta/api/helpers';
+import { auditService } from '../services';
 
 export async function commentRow(req: Request<any, any>, res) {
   res.json(
     await Audit.insert({
       ...req.body,
-      user: (req as any).user?.email,
+      user: (req as any).user,
       op_type: AuditOperationTypes.COMMENT,
     })
   );
 }
 
 export async function auditRowUpdate(req: Request<any, any>, res) {
-  const model = await Model.getByIdOrName({ id: req.body.fk_model_id });
   res.json(
-    await Audit.insert({
-      fk_model_id: req.body.fk_model_id,
-      row_id: req.params.rowId,
-      op_type: AuditOperationTypes.DATA,
-      op_sub_type: AuditOperationSubTypes.UPDATE,
-      description: DOMPurify.sanitize(
-        `Table ${model.table_name} : field ${req.body.column_name} got changed from  ${req.body.prev_value} to ${req.body.value}`
-      ),
-      details: DOMPurify.sanitize(`<span class="">${req.body.column_name}</span>
-  : <span class="text-decoration-line-through red px-2 lighten-4 black--text">${req.body.prev_value}</span>
-  <span class="black--text green lighten-4 px-2">${req.body.value}</span>`),
-      ip: (req as any).clientIp,
-      user: (req as any).user?.email,
+    await auditService.auditRowUpdate({
+      rowId: req.params.rowId,
+      body: {
+        ...req.body,
+      },
     })
   );
 }

packages/nocodb/src/lib/controllers/cacheController.ts

@@ -1,9 +1,9 @@
 import catchError from '../meta/helpers/catchError';
-import NocoCache from '../cache/NocoCache';
 import { Router } from 'express';
+import { cacheService } from '../services';
 
 export async function cacheGet(_, res) {
-  const data = await NocoCache.export();
+  const data = await cacheService.cacheGet();
   res.set({
     'Content-Type': 'application/json',
     'Content-Disposition': `attachment; filename="cache-export.json"`,
@@ -12,7 +12,7 @@ export async function cacheGet(_, res) {
 }
 
 export async function cacheDelete(_, res) {
-  return res.json(await NocoCache.destroy());
+  return res.json(await cacheService.cacheDelete());
 }
 
 const router = Router();

packages/nocodb/src/lib/controllers/dataApis/helpers.ts

@@ -36,7 +36,8 @@ export async function getViewAndModelFromRequestByAliasOrId(
   return { model, view };
 }
 
-export async function extractXlsxData(view: View, req: Request) {
+export async function extractXlsxData(param: {view: View, query:any}) {
+  const { view, query } = param;
   const base = await Base.get(view.base_id);
 
   await view.getModelWithInfo();

packages/nocodb/src/lib/controllers/dataController/export.ts

@@ -0,0 +1,75 @@
+import { Request, Response, Router } from 'express';
+import * as XLSX from 'xlsx';
+import ncMetaAclMw from '../../helpers/ncMetaAclMw';
+import {
+  extractCsvData,
+  extractXlsxData,
+  getViewAndModelFromRequestByAliasOrId,
+} from './helpers';
+import apiMetrics from '../../helpers/apiMetrics';
+import View from '../../../models/View';
+
+async function excelDataExport(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+  let targetView = view;
+  if (!targetView) {
+    targetView = await View.getDefaultView(model.id);
+  }
+  const { offset, elapsed, data } = await extractXlsxData(targetView, req);
+  const wb = XLSX.utils.book_new();
+  XLSX.utils.book_append_sheet(wb, data, targetView.title);
+  const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' });
+  res.set({
+    'Access-Control-Expose-Headers': 'nc-export-offset',
+    'nc-export-offset': offset,
+    'nc-export-elapsed-time': elapsed,
+    'Content-Disposition': `attachment; filename="${encodeURI(
+      targetView.title
+    )}-export.xlsx"`,
+  });
+  res.end(buf);
+}
+
+async function csvDataExport(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+  let targetView = view;
+  if (!targetView) {
+    targetView = await View.getDefaultView(model.id);
+  }
+  const { offset, elapsed, data } = await extractCsvData(targetView, req);
+
+  res.set({
+    'Access-Control-Expose-Headers': 'nc-export-offset',
+    'nc-export-offset': offset,
+    'nc-export-elapsed-time': elapsed,
+    'Content-Disposition': `attachment; filename="${encodeURI(
+      targetView.title
+    )}-export.csv"`,
+  });
+  res.send(data);
+}
+
+const router = Router({ mergeParams: true });
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/export/csv',
+  apiMetrics,
+  ncMetaAclMw(csvDataExport, 'exportCsv')
+);
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/csv',
+  apiMetrics,
+  ncMetaAclMw(csvDataExport, 'exportCsv')
+);
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/export/excel',
+  apiMetrics,
+  ncMetaAclMw(excelDataExport, 'exportExcel')
+);
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/excel',
+  apiMetrics,
+  ncMetaAclMw(excelDataExport, 'exportExcel')
+);
+
+export default router;

packages/nocodb/src/lib/controllers/tableController.ts

@@ -22,7 +22,7 @@ export async function tableList(req: Request, res: Response<TableListType>) {
 }
 
 export async function tableCreate(req: Request<any, any, TableReqType>, res) {
-  const result = tableService.createTable({
+  const result = tableService.tableCreate({
     projectId: req.params.projectId,
     baseId: req.params.baseId,
     table: req.body,

packages/nocodb/src/lib/services/auditService.ts

@@ -0,0 +1,67 @@
+import {
+  AuditRowUpdatePayloadType,
+  CommentRowPayloadType,
+} from 'nocodb-sdk/build/main/lib/CustomAPI';
+import Audit from '../models/Audit';
+import { AuditOperationSubTypes, AuditOperationTypes } from 'nocodb-sdk';
+import Model from '../models/Model';
+import { PagedResponseImpl } from '../meta/helpers/PagedResponse';
+
+import DOMPurify from 'isomorphic-dompurify';
+
+export async function commentRow(param: {
+  rowId: string;
+  body: CommentRowPayloadType;
+  user: any;
+}) {
+  return await Audit.insert({
+    ...param.body,
+    user: param.user?.email,
+    op_type: AuditOperationTypes.COMMENT,
+  });
+}
+
+export async function auditRowUpdate(param: {
+  rowId: string;
+  body: AuditRowUpdatePayloadType;
+}) {
+  const model = await Model.getByIdOrName({ id: param.body.fk_model_id });
+  return await Audit.insert({
+    fk_model_id: param.body.fk_model_id,
+    row_id: param.rowId,
+    op_type: AuditOperationTypes.DATA,
+    op_sub_type: AuditOperationSubTypes.UPDATE,
+    description: DOMPurify.sanitize(
+      `Table ${model.table_name} : field ${param.body.column_name} got changed from  ${param.body.prev_value} to ${param.body.value}`
+    ),
+    details: DOMPurify.sanitize(`<span class="">${param.body.column_name}</span>
+  : <span class="text-decoration-line-through red px-2 lighten-4 black--text">${param.body.prev_value}</span>
+  <span class="black--text green lighten-4 px-2">${param.body.value}</span>`),
+    ip: (param as any).clientIp,
+    user: (param as any).user?.email,
+  });
+}
+
+export async function commentList(param: { query: any }) {
+  return await Audit.commentsList(param.query);
+}
+
+export async function auditList(param: { query: any; projectId: string }) {
+  return new PagedResponseImpl(
+    await Audit.projectAuditList(param.projectId, param.query),
+    {
+      count: await Audit.projectAuditCount(param.projectId),
+      ...param.query,
+    }
+  );
+}
+
+export async function commentsCount(param: {
+  fk_model_id: string;
+  ids: string[];
+}) {
+  return await Audit.commentsCount({
+    fk_model_id: param.fk_model_id as string,
+    ids: param.ids as string[],
+  });
+}

packages/nocodb/src/lib/services/cacheService.ts

@@ -0,0 +1,10 @@
+import NocoCache from '../cache/NocoCache';
+
+export async function cacheGet() {
+  return  await NocoCache.export();
+}
+
+export async function cacheDelete() {
+  await NocoCache.destroy()
+  return true
+}

packages/nocodb/src/lib/services/dataService/export.ts

@@ -0,0 +1,54 @@
+import { Request, Response, Router } from 'express';
+import { isSystemColumn } from 'nocodb-sdk'
+import * as XLSX from 'xlsx';
+import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'
+import ncMetaAclMw from '../../helpers/ncMetaAclMw';
+import { extractXlsxData, serializeCellValue } from '../../meta/api/dataApis/helpers'
+import {
+  extractCsvData, getViewAndModelByAliasOrId,
+  getViewAndModelFromRequestByAliasOrId, PathParams,
+} from './helpers'
+import apiMetrics from '../../helpers/apiMetrics';
+import View from '../../../models/View';
+
+async function excelDataExport(param:PathParams&{
+  query: any;
+}) {
+  const { model, view } = await getViewAndModelByAliasOrId(param);
+  let targetView = view;
+  if (!targetView) {
+    targetView = await View.getDefaultView(model.id);
+  }
+  const { offset, elapsed, data } = await extractXlsxData({view: targetView, query:req.query });
+  const wb = XLSX.utils.book_new();
+  XLSX.utils.book_append_sheet(wb, data, targetView.title);
+  const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' });
+  res.set({
+    'Access-Control-Expose-Headers': 'nc-export-offset',
+    'nc-export-offset': offset,
+    'nc-export-elapsed-time': elapsed,
+    'Content-Disposition': `attachment; filename="${encodeURI(
+      targetView.title
+    )}-export.xlsx"`,
+  });
+  res.end(buf);
+}
+
+async function csvDataExport(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+  let targetView = view;
+  if (!targetView) {
+    targetView = await View.getDefaultView(model.id);
+  }
+  const { offset, elapsed, data } = await extractCsvData(targetView, req);
+
+  res.set({
+    'Access-Control-Expose-Headers': 'nc-export-offset',
+    'nc-export-offset': offset,
+    'nc-export-elapsed-time': elapsed,
+    'Content-Disposition': `attachment; filename="${encodeURI(
+      targetView.title
+    )}-export.csv"`,
+  });
+  res.send(data);
+}

packages/nocodb/src/lib/services/dataService/helpers.ts

@@ -1,6 +1,17 @@
+import { Request } from 'express'
+import { nocoExecute } from 'nc-help'
+import { isSystemColumn, UITypes } from 'nocodb-sdk'
+import * as XLSX from 'xlsx'
+import { BaseModelSqlv2 } from '../../db/sql-data-mapper/lib/sql/BaseModelSqlv2'
+import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'
 import { NcError } from '../../meta/helpers/catchError'
 import { Model, View } from '../../models'
+import Base from '../../models/Base'
+import Column from '../../models/Column'
+import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'
+import LookupColumn from '../../models/LookupColumn'
 import Project from '../../models/Project'
+import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'
 
 
 export interface PathParams {
@@ -29,3 +40,286 @@ export async function getViewAndModelByAliasOrId(param: {
   if (!model) NcError.notFound('Table not found');
   return { model, view };
 }
+
+
+
+export async function extractXlsxData(view: View, req: Request) {
+  const base = await Base.get(view.base_id);
+
+  await view.getModelWithInfo();
+  await view.getColumns();
+
+  view.model.columns = view.columns
+    .filter((c) => c.show)
+    .map(
+      (c) =>
+        new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any)
+    )
+    .filter((column) => !isSystemColumn(column) || view.show_system_fields);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: view.model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req);
+
+  const fields = req.query.fields as string[];
+
+  const data = XLSX.utils.json_to_sheet(dbRows, { header: fields });
+
+  return { offset, dbRows, elapsed, data };
+}
+
+export async function extractCsvData(view: View, req: Request) {
+  const base = await Base.get(view.base_id);
+  const fields = req.query.fields;
+
+  await view.getModelWithInfo();
+  await view.getColumns();
+
+  view.model.columns = view.columns
+    .filter((c) => c.show)
+    .map(
+      (c) =>
+        new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any)
+    )
+    .filter((column) => !isSystemColumn(column) || view.show_system_fields);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: view.model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req);
+
+  const data = papaparse.unparse(
+    {
+      fields: view.model.columns
+        .sort((c1, c2) =>
+          Array.isArray(fields)
+            ? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any)
+            : 0
+        )
+        .filter(
+          (c) =>
+            !fields || !Array.isArray(fields) || fields.includes(c.title as any)
+        )
+        .map((c) => c.title),
+      data: dbRows,
+    },
+    {
+      escapeFormulae: true,
+    }
+  );
+
+  return { offset, dbRows, elapsed, data };
+}
+
+
+async function getDbRows(baseModel, view: View, req: Request) {
+  let offset = +req.query.offset || 0;
+  const limit = 100;
+  // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024;
+  const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000;
+  const dbRows = [];
+  const startTime = process.hrtime();
+  let elapsed, temp;
+
+  const listArgs: any = { ...req.query };
+  try {
+    listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
+  } catch (e) {}
+  try {
+    listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
+  } catch (e) {}
+
+  for (
+    elapsed = 0;
+    elapsed < timeout;
+    offset += limit,
+      temp = process.hrtime(startTime),
+      elapsed = temp[0] * 1000 + temp[1] / 1000000
+  ) {
+    const rows = await nocoExecute(
+      await getAst({
+        query: req.query,
+        includePkByDefault: false,
+        model: view.model,
+        view,
+      }),
+      await baseModel.list({ ...listArgs, offset, limit }),
+      {},
+      req.query
+    );
+
+    if (!rows?.length) {
+      offset = -1;
+      break;
+    }
+
+    for (const row of rows) {
+      const dbRow = { ...row };
+
+      for (const column of view.model.columns) {
+        if (isSystemColumn(column) && !view.show_system_fields) continue;
+        dbRow[column.title] = await serializeCellValue({
+          value: row[column.title],
+          column,
+          siteUrl: req['ncSiteUrl'],
+        });
+      }
+      dbRows.push(dbRow);
+    }
+  }
+  return { offset, dbRows, elapsed };
+}
+
+export async function serializeCellValue({
+                                           value,
+                                           column,
+                                           siteUrl,
+                                         }: {
+  column?: Column;
+  value: any;
+  siteUrl: string;
+}) {
+  if (!column) {
+    return value;
+  }
+
+  if (!value) return value;
+
+  switch (column?.uidt) {
+    case UITypes.Attachment: {
+      let data = value;
+      try {
+        if (typeof value === 'string') {
+          data = JSON.parse(value);
+        }
+      } catch {}
+
+      return (data || []).map(
+        (attachment) =>
+          `${encodeURI(attachment.title)}(${encodeURI(
+            attachment.path ? `${siteUrl}/${attachment.path}` : attachment.url
+          )})`
+      );
+    }
+    case UITypes.Lookup:
+    {
+      const colOptions = await column.getColOptions<LookupColumn>();
+      const lookupColumn = await colOptions.getLookupColumn();
+      return (
+        await Promise.all(
+          [...(Array.isArray(value) ? value : [value])].map(async (v) =>
+            serializeCellValue({
+              value: v,
+              column: lookupColumn,
+              siteUrl,
+            })
+          )
+        )
+      ).join(', ');
+    }
+      break;
+    case UITypes.LinkToAnotherRecord:
+    {
+      const colOptions =
+        await column.getColOptions<LinkToAnotherRecordColumn>();
+      const relatedModel = await colOptions.getRelatedTable();
+      await relatedModel.getColumns();
+      return [...(Array.isArray(value) ? value : [value])]
+        .map((v) => {
+          return v[relatedModel.displayValue?.title];
+        })
+        .join(', ');
+    }
+      break;
+    default:
+      if (value && typeof value === 'object') {
+        return JSON.stringify(value);
+      }
+      return value;
+  }
+}
+
+export async function getColumnByIdOrName(
+  columnNameOrId: string,
+  model: Model
+) {
+  const column = (await model.getColumns()).find(
+    (c) =>
+      c.title === columnNameOrId ||
+      c.id === columnNameOrId ||
+      c.column_name === columnNameOrId
+  );
+
+  if (!column)
+    NcError.notFound(`Column with id/name '${columnNameOrId}' is not found`);
+
+  return column;
+}
+
+
+
+async function getDbRows(param: { baseModel:BaseModelSqlv2, view: View, query: any; siteUrl: string; }) {
+  const { baseModel, view, query = {}, siteUrl } = param;
+  let offset = +query.offset || 0;
+  const limit = 100;
+  // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024;
+  const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000;
+  const dbRows = [];
+  const startTime = process.hrtime();
+  let elapsed, temp;
+
+  const listArgs: any = { ...query };
+  try {
+    listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
+  } catch (e) {}
+  try {
+    listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
+  } catch (e) {}
+
+  for (
+    elapsed = 0;
+    elapsed < timeout;
+    offset += limit,
+      temp = process.hrtime(startTime),
+      elapsed = temp[0] * 1000 + temp[1] / 1000000
+  ) {
+    const rows = await nocoExecute(
+      await getAst({
+        query: query,
+        includePkByDefault: false,
+        model: view.model,
+        view,
+      }),
+      await baseModel.list({ ...listArgs, offset, limit }),
+      {},
+      query
+    );
+
+    if (!rows?.length) {
+      offset = -1;
+      break;
+    }
+
+    for (const row of rows) {
+      const dbRow = { ...row };
+
+      for (const column of view.model.columns) {
+        if (isSystemColumn(column) && !view.show_system_fields) continue;
+        dbRow[column.title] = await serializeCellValue({
+          value: row[column.title],
+          column,
+          siteUrl,
+        });
+      }
+      dbRows.push(dbRow);
+    }
+  }
+  return { offset, dbRows, elapsed };
+}

packages/nocodb/src/lib/services/dataService/index.ts

@@ -1,4 +1,4 @@
-import { nocoExecute } from 'nc-help/dist/module/NocoExecute';
+import { nocoExecute } from 'nc-help';
 import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst';
 import { NcError } from '../../meta/helpers/catchError';
 import { PagedResponseImpl } from '../../meta/helpers/PagedResponse';

packages/nocodb/src/lib/services/index.ts

@@ -28,3 +28,5 @@ export * as attachmentService from './attachmentService';
 export * as hookFilterService from './hookFilterService';
 export * as dataService from './dataService';
 export * as bulkDataService from './dataService/bulkData';
+export * as cacheService from './cacheService';
+export * as auditService from './auditService';