Browse Source

feat: nested field filtering (#1856)

* feat: allow user to include/exclude nested fields and by default only extract pk&pv

Signed-off-by: Pranav C <pranavxc@gmail.com>

* chore: api metrics

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: add missing await and code cleanup

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: corrections in ast population

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: map default primary value column on table create and include field param for belongsTo column

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: old data list api response correction and count api

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/1901/head
Pranav C 3 years ago committed by GitHub
parent
commit
60979cce01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      packages/noco-docs/content/en/setup-and-usages/usage-information.md
  2. 25203
      packages/nocodb/package-lock.json
  3. 2
      packages/nocodb/package.json
  4. 62
      packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts
  5. 99
      packages/nocodb/src/lib/dataMapper/lib/sql/helpers/getAst.ts
  6. 8
      packages/nocodb/src/lib/noco-models/Model.ts
  7. 14
      packages/nocodb/src/lib/noco/Noco.ts
  8. 6
      packages/nocodb/src/lib/noco/meta/api/dataApis/bulkDataAliasApis.ts
  9. 23
      packages/nocodb/src/lib/noco/meta/api/dataApis/dataAliasApis.ts
  10. 3
      packages/nocodb/src/lib/noco/meta/api/dataApis/dataAliasExportApis.ts
  11. 8
      packages/nocodb/src/lib/noco/meta/api/dataApis/dataAliasNestedApis.ts
  12. 87
      packages/nocodb/src/lib/noco/meta/api/dataApis/dataApis.ts
  13. 8
      packages/nocodb/src/lib/noco/meta/api/dataApis/helpers.ts
  14. 102
      packages/nocodb/src/lib/noco/meta/api/dataApis/oldDataApis.ts
  15. 6
      packages/nocodb/src/lib/noco/meta/api/index.ts
  16. 34
      packages/nocodb/src/lib/noco/meta/api/publicApis/publicDataApis.ts
  17. 33
      packages/nocodb/src/lib/noco/meta/api/publicApis/publicDataExportApis.ts
  18. 2
      packages/nocodb/src/lib/noco/meta/api/swagger/helpers/templates/params.ts
  19. 2
      packages/nocodb/src/lib/noco/meta/api/tableApis.ts
  20. 19
      packages/nocodb/src/lib/noco/meta/helpers/apiMetrics.ts

2
packages/noco-docs/content/en/setup-and-usages/usage-information.md

@ -42,7 +42,7 @@ Here is an example :
"os_release" : "5.10.25-linuxkit",
"node_version" : "14.18.2",
"docker" : "true",
"xc_version" : "0.84.15",
"xc_version" : "a0885e8e6a38d9fbb5d39e7d04a44da7773d4f",
"env" : "dev",
}

25203
packages/nocodb/package-lock.json generated

File diff suppressed because it is too large Load Diff

2
packages/nocodb/package.json

@ -150,7 +150,7 @@
"mysql2": "^2.2.5",
"nanoid": "^3.1.20",
"nc-common": "0.0.6",
"nc-help": "^0.2.44",
"nc-help": "0.2.46",
"nc-lib-gui": "0.90.7",
"nc-plugin": "^0.1.1",
"ncp": "^2.0.0",

62
packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts

@ -21,7 +21,6 @@ import View from '../../../noco-models/View';
import {
AuditOperationSubTypes,
AuditOperationTypes,
isSystemColumn,
RelationTypes,
SortType,
UITypes,
@ -288,69 +287,8 @@ class BaseModelSqlv2 {
return ((await qb) as any).count;
}
public async defaultResolverReq(
query?: any,
extractOnlyPrimaries = false,
includePkByDefault = true
) {
await this.model.getColumns();
if (extractOnlyPrimaries) {
return {
[this.model.primaryKey.title]: 1,
[this.model.primaryValue.title]: 1
};
}
let fields = query?.fields || query?.f;
if (fields && fields !== '*') {
fields = Array.isArray(fields) ? fields : fields.split(',');
} else {
fields = null;
}
let allowedCols = null;
if (this.viewId)
allowedCols = (await View.getColumns(this.viewId)).reduce(
(o, c) => ({
...o,
[c.fk_column_id]: c.show
}),
{}
);
const view = await View.get(this.viewId);
return this.model.getColumns().then(columns =>
Promise.resolve(
columns.reduce(
(obj, col) => ({
...obj,
[col.title]:
allowedCols && (!includePkByDefault || !col.pk)
? allowedCols[col.id] &&
(!isSystemColumn(col) || view.show_system_fields) &&
(!fields?.length || fields.includes(col.title))
: fields?.length
? fields.includes(col.title)
: 1
}),
{}
)
)
);
}
async multipleHmList({ colId, ids }, args?: { limit?; offset? }) {
try {
// const {
// where,
// limit,
// offset,
// conditionGraph,
// sort
// // ...restArgs
// } = this.dbModels[child]._getChildListArgs(args);
// let { fields } = restArgs;
// todo: get only required fields
let fields = '*';

99
packages/nocodb/src/lib/dataMapper/lib/sql/helpers/getAst.ts

@ -0,0 +1,99 @@
import View from '../../../../noco-models/View';
import { isSystemColumn, UITypes } from 'nocodb-sdk';
import Model from '../../../../noco-models/Model';
import LinkToAnotherRecordColumn from '../../../../noco-models/LinkToAnotherRecordColumn';
const getAst = async ({
query,
extractOnlyPrimaries = false,
includePkByDefault = true,
model,
view
}: {
query?: RequestQuery;
extractOnlyPrimaries?: boolean;
includePkByDefault?: boolean;
model: Model;
view?: View;
}) => {
if (!model.columns?.length) await model.getColumns();
if (extractOnlyPrimaries) {
return {
[model.primaryKey.title]: 1,
[model.primaryValue.title]: 1
};
}
let fields = query?.fields || query?.f;
if (fields && fields !== '*') {
fields = Array.isArray(fields) ? fields : fields.split(',');
} else {
fields = null;
}
let allowedCols = null;
if (view)
allowedCols = (await View.getColumns(view.id)).reduce(
(o, c) => ({
...o,
[c.fk_column_id]: c.show
}),
{}
);
return model.columns.reduce(async (obj, col) => {
let value: number | boolean | { [key: string]: any } = 1;
const nestedFields =
query?.nested?.[col.title]?.fields || query?.nested?.[col.title]?.f;
if (nestedFields && nestedFields !== '*') {
if (col.uidt === UITypes.LinkToAnotherRecord) {
const model = await col
.getColOptions<LinkToAnotherRecordColumn>()
.then(colOpt => colOpt.getRelatedTable());
value = await getAst({
model,
query: query?.nested?.[col.title]
});
} else {
value = (Array.isArray(fields) ? fields : fields.split(',')).reduce(
(o, f) => ({ ...o, [f]: 1 }),
{}
);
}
} else if (col.uidt === UITypes.LinkToAnotherRecord) {
const model = await col
.getColOptions<LinkToAnotherRecordColumn>()
.then(colOpt => colOpt.getRelatedTable());
value = await getAst({
model,
query: query?.nested,
extractOnlyPrimaries: true
});
}
return {
...(await obj),
[col.title]:
allowedCols && (!includePkByDefault || !col.pk)
? allowedCols[col.id] &&
(!isSystemColumn(col) || view.show_system_fields) &&
(!fields?.length || fields.includes(col.title)) &&
value
: fields?.length
? fields.includes(col.title)
: value
};
}, Promise.resolve({}));
};
type RequestQuery = {
[fields in 'f' | 'fields']?: string | string[];
} & {
nested?: {
[field: string]: RequestQuery;
};
};
export default getAst;

8
packages/nocodb/src/lib/noco-models/Model.ts

@ -51,14 +51,6 @@ export default class Model implements TableType {
columnsById?: { [id: string]: Column };
views?: View[];
// private static baseModels: {
// [baseId: string]: {
// [dbAlias: string]: {
// [tableIdOrName: string]: BaseModelSqlv2;
// };
// };
// } = {};
constructor(data: Partial<TableType | Model>) {
Object.assign(this, data);
}

14
packages/nocodb/src/lib/noco/Noco.ts

@ -568,14 +568,28 @@ export default class Noco {
}
this.config.auth.jwt.secret = secret;
}
let serverId = (
await Noco._ncMeta.metaGet('', '', 'nc_store', {
key: 'nc_server_id'
})
)?.value;
if (!serverId) {
await Noco._ncMeta.metaInsert('', '', 'nc_store', {
key: 'nc_server_id',
value: serverId = Tele.id
});
}
process.env.NC_SERVER_UUID = serverId;
}
public static get ncMeta(): NcMetaIO {
return this._ncMeta;
}
public get ncMeta(): NcMetaIO {
return Noco._ncMeta;
}
public static getConfig(): NcConfig {
return Noco.config;
}

6
packages/nocodb/src/lib/noco/meta/api/dataApis/bulkDataAliasApis.ts

@ -4,6 +4,7 @@ import Base from '../../../../noco-models/Base';
import NcConnectionMgrv2 from '../../../common/NcConnectionMgrv2';
import ncMetaAclMw from '../../helpers/ncMetaAclMw';
import { getViewAndModelFromRequestByAliasOrId } from './helpers';
import apiMetrics from '../../helpers/apiMetrics';
async function bulkDataInsert(req: Request, res: Response) {
const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
@ -72,22 +73,27 @@ const router = Router({ mergeParams: true });
router.post(
'/api/v1/db/data/bulk/:orgs/:projectName/:tableName',
apiMetrics,
ncMetaAclMw(bulkDataInsert, 'bulkDataInsert')
);
router.patch(
'/api/v1/db/data/bulk/:orgs/:projectName/:tableName',
apiMetrics,
ncMetaAclMw(bulkDataUpdate, 'bulkDataUpdate')
);
router.patch(
'/api/v1/db/data/bulk/:orgs/:projectName/:tableName/all',
apiMetrics,
ncMetaAclMw(bulkDataUpdateAll, 'bulkDataUpdateAll')
);
router.delete(
'/api/v1/db/data/bulk/:orgs/:projectName/:tableName',
apiMetrics,
ncMetaAclMw(bulkDataDelete, 'bulkDataDelete')
);
router.delete(
'/api/v1/db/data/bulk/:orgs/:projectName/:tableName/all',
apiMetrics,
ncMetaAclMw(bulkDataDeleteAll, 'bulkDataDeleteAll')
);

23
packages/nocodb/src/lib/noco/meta/api/dataApis/dataAliasApis.ts

@ -7,6 +7,8 @@ import { PagedResponseImpl } from '../../helpers/PagedResponse';
import View from '../../../../noco-models/View';
import ncMetaAclMw from '../../helpers/ncMetaAclMw';
import { getViewAndModelFromRequestByAliasOrId } from './helpers';
import apiMetrics from '../../helpers/apiMetrics';
import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
async function dataList(req: Request, res: Response) {
const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
@ -86,7 +88,7 @@ async function getDataList(model, view: View, req) {
dbDriver: NcConnectionMgrv2.get(base)
});
const requestObj = await baseModel.defaultResolverReq(req.query);
const requestObj = await getAst({ model, query: req.query, view });
const listArgs: any = { ...req.query };
try {
@ -129,7 +131,7 @@ async function getFindOne(model, view: View, req) {
} catch (e) {}
return await nocoExecute(
await baseModel.defaultResolverReq(),
await getAst({ model, query: args, view }),
await baseModel.findOne(args),
{},
{}
@ -149,7 +151,7 @@ async function dataRead(req: Request, res: Response) {
res.json(
await nocoExecute(
await baseModel.defaultResolverReq(),
await getAst({ model, query: req.query, view }),
await baseModel.readByPk(req.params.rowId),
{},
{}
@ -162,70 +164,85 @@ const router = Router({ mergeParams: true });
// table data crud apis
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName',
apiMetrics,
ncMetaAclMw(dataList, 'dataList')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/find-one',
apiMetrics,
ncMetaAclMw(dataFindOne, 'dataFindOne')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/count',
apiMetrics,
ncMetaAclMw(dataCount, 'dataCount')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/count',
apiMetrics,
ncMetaAclMw(dataCount, 'dataCount')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/:rowId',
apiMetrics,
ncMetaAclMw(dataRead, 'dataRead')
);
router.patch(
'/api/v1/db/data/:orgs/:projectName/:tableName/:rowId',
apiMetrics,
ncMetaAclMw(dataUpdate, 'dataUpdate')
);
router.delete(
'/api/v1/db/data/:orgs/:projectName/:tableName/:rowId',
apiMetrics,
ncMetaAclMw(dataDelete, 'dataDelete')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName',
apiMetrics,
ncMetaAclMw(dataList, 'dataList')
);
// table view data crud apis
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName',
apiMetrics,
ncMetaAclMw(dataList, 'dataList')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/find-one',
apiMetrics,
ncMetaAclMw(dataFindOne, 'dataFindOne')
);
router.post(
'/api/v1/db/data/:orgs/:projectName/:tableName',
apiMetrics,
ncMetaAclMw(dataInsert, 'dataInsert')
);
router.post(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName',
apiMetrics,
ncMetaAclMw(dataInsert, 'dataInsert')
);
router.patch(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId',
apiMetrics,
ncMetaAclMw(dataUpdate, 'dataUpdate')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId',
apiMetrics,
ncMetaAclMw(dataRead, 'dataRead')
);
router.delete(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId',
apiMetrics,
ncMetaAclMw(dataDelete, 'dataDelete')
);

3
packages/nocodb/src/lib/noco/meta/api/dataApis/dataAliasExportApis.ts

@ -4,6 +4,7 @@ import {
extractCsvData,
getViewAndModelFromRequestByAliasOrId
} from './helpers';
import apiMetrics from '../../helpers/apiMetrics';
async function csvDataExport(req: Request, res: Response) {
const { view } = await getViewAndModelFromRequestByAliasOrId(req);
@ -23,10 +24,12 @@ const router = Router({ mergeParams: true });
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/export/csv',
apiMetrics,
ncMetaAclMw(csvDataExport, 'exportCsv')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/csv',
apiMetrics,
ncMetaAclMw(csvDataExport, 'exportCsv')
);

8
packages/nocodb/src/lib/noco/meta/api/dataApis/dataAliasNestedApis.ts

@ -6,6 +6,7 @@ import { PagedResponseImpl } from '../../helpers/PagedResponse';
import ncMetaAclMw from '../../helpers/ncMetaAclMw';
import { getViewAndModelFromRequestByAliasOrId } from './helpers';
import { NcError } from '../../helpers/catchError';
import apiMetrics from '../../helpers/apiMetrics';
export async function mmList(req: Request, res: Response, next) {
const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
@ -256,32 +257,39 @@ const router = Router({ mergeParams: true });
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/mm/:columnName/exclude',
apiMetrics,
ncMetaAclMw(mmExcludedList, 'mmExcludedList')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/hm/:columnName/exclude',
apiMetrics,
ncMetaAclMw(hmExcludedList, 'hmExcludedList')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/bt/:columnName/exclude',
apiMetrics,
ncMetaAclMw(btExcludedList, 'btExcludedList')
);
router.post(
'/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/:relationType/:columnName/:refRowId',
apiMetrics,
ncMetaAclMw(relationDataAdd, 'relationDataAdd')
);
router.delete(
'/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/:relationType/:columnName/:refRowId',
apiMetrics,
ncMetaAclMw(relationDataRemove, 'relationDataRemove')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/mm/:columnName',
apiMetrics,
ncMetaAclMw(mmList, 'mmList')
);
router.get(
'/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/hm/:columnName',
apiMetrics,
ncMetaAclMw(hmList, 'hmList')
);

87
packages/nocodb/src/lib/noco/meta/api/dataApis/dataApis.ts

@ -7,6 +7,8 @@ import { PagedResponseImpl } from '../../helpers/PagedResponse';
import View from '../../../../noco-models/View';
import ncMetaAclMw from '../../helpers/ncMetaAclMw';
import { NcError } from '../../helpers/catchError';
import apiMetrics from '../../helpers/apiMetrics';
import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
export async function dataList(req: Request, res: Response, next) {
const view = await View.get(req.params.viewId);
@ -315,24 +317,14 @@ async function dataRead(req: Request, res: Response, next) {
id: model.id,
dbDriver: NcConnectionMgrv2.get(base)
});
const key = `${model.title}Read`;
res.json(
(
await nocoExecute(
{
[key]: await baseModel.defaultResolverReq()
},
{
[key]: async id => {
return await baseModel.readByPk(id);
// return row ? new ctx.types[model.title](row) : null;
}
},
{},
{ nested: { [key]: req.params.rowId } }
)
)?.[key]
await nocoExecute(
await getAst({ model, query: req.query }),
await baseModel.readByPk(req.params.rowId),
{},
{}
)
);
} catch (e) {
console.log(e);
@ -450,10 +442,7 @@ async function getDataList(model, view: View, req) {
dbDriver: NcConnectionMgrv2.get(base)
});
const key = `${model._tn}List`;
const requestObj = {
[key]: await baseModel.defaultResolverReq(req.query)
};
const requestObj = await getAst({ query: req.query, model, view });
const listArgs: any = { ...req.query };
try {
@ -463,18 +452,12 @@ async function getDataList(model, view: View, req) {
listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
} catch (e) {}
const data = (
await nocoExecute(
requestObj,
{
[key]: async args => {
return await baseModel.list(args);
}
},
{},
{ nested: { [key]: listArgs } }
)
)?.[key];
const data = await nocoExecute(
requestObj,
await baseModel.list(listArgs),
{},
listArgs
);
const count = await baseModel.count(listArgs);
@ -539,7 +522,7 @@ async function relationDataAdd(req, res) {
const router = Router({ mergeParams: true });
// router.get('/data/:orgs/:projectName/:tableName', ncMetaAclMw(dataListNew));
// router.get('/data/:orgs/:projectName/:tableName',apiMetrics,ncMetaAclMw(dataListNew));
// router.get(
// '/data/:orgs/:projectName/:tableName/views/:viewName',
// ncMetaAclMw(dataListNew)
@ -558,14 +541,38 @@ const router = Router({ mergeParams: true });
// ncMetaAclMw(dataDeleteNew)
// );
router.get('/data/:viewId/', ncMetaAclMw(dataList, 'dataList'));
router.post('/data/:viewId/', ncMetaAclMw(dataInsert, 'dataInsert'));
router.get('/data/:viewId/:rowId', ncMetaAclMw(dataRead, 'dataRead'));
router.patch('/data/:viewId/:rowId', ncMetaAclMw(dataUpdate, 'dataUpdate'));
router.delete('/data/:viewId/:rowId', ncMetaAclMw(dataDelete, 'dataDelete'));
router.get('/data/:viewId/', apiMetrics, ncMetaAclMw(dataList, 'dataList'));
router.post(
'/data/:viewId/',
apiMetrics,
ncMetaAclMw(dataInsert, 'dataInsert')
);
router.get(
'/data/:viewId/:rowId',
apiMetrics,
ncMetaAclMw(dataRead, 'dataRead')
);
router.patch(
'/data/:viewId/:rowId',
apiMetrics,
ncMetaAclMw(dataUpdate, 'dataUpdate')
);
router.delete(
'/data/:viewId/:rowId',
apiMetrics,
ncMetaAclMw(dataDelete, 'dataDelete')
);
router.get('/data/:viewId/:rowId/mm/:colId', ncMetaAclMw(mmList, 'mmList'));
router.get('/data/:viewId/:rowId/hm/:colId', ncMetaAclMw(hmList, 'hmList'));
router.get(
'/data/:viewId/:rowId/mm/:colId',
apiMetrics,
ncMetaAclMw(mmList, 'mmList')
);
router.get(
'/data/:viewId/:rowId/hm/:colId',
apiMetrics,
ncMetaAclMw(hmList, 'hmList')
);
router.get(
'/data/:viewId/:rowId/mm/:colId/exclude',

8
packages/nocodb/src/lib/noco/meta/api/dataApis/helpers.ts

@ -13,6 +13,7 @@ import LookupColumn from '../../../../noco-models/LookupColumn';
import LinkToAnotherRecordColumn from '../../../../noco-models/LinkToAnotherRecordColumn';
import papaparse from 'papaparse';
import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
export async function getViewAndModelFromRequestByAliasOrId(
req:
| Request<{ projectName: string; tableName: string; viewName?: string }>
@ -71,7 +72,12 @@ export async function extractCsvData(view: View, req: Request) {
elapsed = temp[0] * 1000 + temp[1] / 1000000
) {
const rows = await nocoExecute(
await baseModel.defaultResolverReq(req.query, false, false),
await getAst({
query: req.query,
includePkByDefault: false,
model: view.model,
view
}),
await baseModel.list({ ...req.query, offset, limit }),
{},
req.query

102
packages/nocodb/src/lib/noco/meta/api/dataApis/oldDataApis.ts

@ -3,15 +3,66 @@ import Model from '../../../../noco-models/Model';
import { nocoExecute } from 'nc-help';
import Base from '../../../../noco-models/Base';
import NcConnectionMgrv2 from '../../../common/NcConnectionMgrv2';
import { PagedResponseImpl } from '../../helpers/PagedResponse';
import View from '../../../../noco-models/View';
import ncMetaAclMw from '../../helpers/ncMetaAclMw';
import Project from '../../../../noco-models/Project';
import { NcError } from '../../helpers/catchError';
import apiMetrics from '../../helpers/apiMetrics';
import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
export async function dataList(req: Request, res: Response) {
const { model, view } = await getViewAndModelFromRequest(req);
res.json(await getDataList(model, view, req));
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: NcConnectionMgrv2.get(base)
});
const requestObj = await getAst({
query: req.query,
model,
view
});
const listArgs: any = { ...req.query };
try {
listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
} catch (e) {}
try {
listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
} catch (e) {}
const data = await nocoExecute(
requestObj,
await baseModel.list(listArgs),
{},
listArgs
);
res.json(data);
}
export async function dataCount(req: Request, res: Response) {
const { model, view } = await getViewAndModelFromRequest(req);
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: NcConnectionMgrv2.get(base)
});
const listArgs: any = { ...req.query };
try {
listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
} catch (e) {}
const count = await baseModel.count(listArgs);
res.json({
count
});
}
async function dataInsert(req: Request, res: Response) {
@ -52,39 +103,7 @@ async function dataDelete(req: Request, res: Response) {
res.json(await baseModel.delByPk(req.params.rowId, null, req));
}
async function getDataList(model, view: View, req) {
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: NcConnectionMgrv2.get(base)
});
const requestObj = await baseModel.defaultResolverReq(req.query);
const listArgs: any = { ...req.query };
try {
listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
} catch (e) {}
try {
listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
} catch (e) {}
const data = await nocoExecute(
requestObj,
await baseModel.list(listArgs),
{},
listArgs
);
const count = await baseModel.count(listArgs);
return new PagedResponseImpl(data, {
...req.query,
count
});
}
async function getViewAndModelFromRequest(req) {
const project = await Project.getWithInfo(req.params.projectId);
const model = await Model.getByAliasOrId({
@ -115,7 +134,11 @@ async function dataRead(req: Request, res: Response) {
res.json(
await nocoExecute(
await baseModel.defaultResolverReq(),
await getAst({
query: req.query,
model,
view
}),
await baseModel.readByPk(req.params.rowId),
{},
{}
@ -130,20 +153,29 @@ router.get(
ncMetaAclMw(dataList, 'dataList')
);
router.get(
'/nc/:projectId/api/v1/:tableName/count',
ncMetaAclMw(dataCount, 'dataCount')
);
router.post(
'/nc/:projectId/api/v1/:tableName',
apiMetrics,
ncMetaAclMw(dataInsert, 'dataInsert')
);
router.get(
'/nc/:projectId/api/v1/:tableName/:rowId',
apiMetrics,
ncMetaAclMw(dataRead, 'dataRead')
);
router.patch(
'/nc/:projectId/api/v1/:tableName/:rowId',
apiMetrics,
ncMetaAclMw(dataUpdate, 'dataUpdate')
);
router.delete(
'/nc/:projectId/api/v1/:tableName/:rowId',
apiMetrics,
ncMetaAclMw(dataDelete, 'dataDelete')
);

6
packages/nocodb/src/lib/noco/meta/api/index.ts

@ -95,7 +95,6 @@ export default function(router: Router, server) {
credentials: true
}
});
io.use(function(socket, next) {
passport.authenticate(
'jwt',
@ -110,7 +109,10 @@ export default function(router: Router, server) {
}
)(socket.handshake, {}, next);
}).on('connection', socket => {
const id = getHash(Tele.id + (socket?.handshake as any)?.user?.id);
const id = getHash(
(process.env.NC_SERVER_UUID || Tele.id) +
(socket?.handshake as any)?.user?.id
);
socket.on('page', args => {
Tele.page({ ...args, id });

34
packages/nocodb/src/lib/noco/meta/api/publicApis/publicDataApis.ts

@ -16,6 +16,7 @@ import { nanoid } from 'nanoid';
import { mimeIcons } from '../../../../utils/mimeTypes';
import slash from 'slash';
import { sanitizeUrlPath } from '../attachmentApis';
import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
export async function dataList(req: Request, res: Response) {
try {
@ -49,7 +50,11 @@ export async function dataList(req: Request, res: Response) {
} catch (e) {}
const data = await nocoExecute(
await baseModel.defaultResolverReq(req.query),
await getAst({
query: req.query,
model,
view
}),
await baseModel.list(listArgs),
{},
listArgs
@ -186,23 +191,18 @@ async function relDataList(req, res) {
dbDriver: NcConnectionMgrv2.get(base)
});
const key = `${model.title}List`;
const requestObj = {
[key]: await baseModel.defaultResolverReq(req.query, true)
};
const requestObj = await getAst({
query: req.query,
model,
extractOnlyPrimaries: true
});
const data = (
await nocoExecute(
requestObj,
{
[key]: async args => {
return await baseModel.list(args);
}
},
{},
{ nested: { [key]: req.query } }
)
)?.[key];
const data = await nocoExecute(
requestObj,
await baseModel.list(req.query),
{},
req.query
);
const count = await baseModel.count(req.query);

33
packages/nocodb/src/lib/noco/meta/api/publicApis/publicDataExportApis.ts

@ -10,6 +10,7 @@ import Column from '../../../../noco-models/Column';
import LinkToAnotherRecordColumn from '../../../../noco-models/LinkToAnotherRecordColumn';
import LookupColumn from '../../../../noco-models/LookupColumn';
import catchError, { NcError } from '../../helpers/catchError';
import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
async function exportCsv(req: Request, res: Response) {
const view = await View.getByUUID(req.params.publicDataUuid);
@ -50,10 +51,12 @@ async function exportCsv(req: Request, res: Response) {
dbDriver: NcConnectionMgrv2.get(base)
});
const key = `${model.title}List`;
const requestObj = {
[key]: await baseModel.defaultResolverReq(req.query, false, false)
};
const requestObj = await getAst({
query: req.query,
model,
view,
includePkByDefault: false
});
let offset = +req.query.offset || 0;
const limit = 100;
@ -70,22 +73,12 @@ async function exportCsv(req: Request, res: Response) {
temp = process.hrtime(startTime),
elapsed = temp[0] * 1000 + temp[1] / 1000000
) {
const rows = (
await nocoExecute(
requestObj,
{
[key]: async args => {
return await baseModel.list({ ...args, offset, limit });
}
},
{},
{
nested: {
[key]: listArgs
}
}
)
)?.[key];
const rows = await nocoExecute(
requestObj,
await baseModel.list({ ...listArgs, offset, limit }),
{},
listArgs
);
if (!rows?.length) {
offset = -1;

2
packages/nocodb/src/lib/noco/meta/api/swagger/helpers/templates/params.ts

@ -184,6 +184,8 @@ export const getNestedParams = async (
nestedFieldParam(column.title),
nestedSortParam(column.title)
];
} else {
return [...(await paramsArr), nestedFieldParam(column.title)];
}
}

2
packages/nocodb/src/lib/noco/meta/api/tableApis.ts

@ -157,7 +157,7 @@ export async function tableCreate(req: Request<any, any, TableReqType>, res) {
ip: (req as any).clientIp
}).then(() => {});
mapDefaultPrimaryValue(req.body.columns);
mapDefaultPrimaryValue(columns);
Tele.emit('evt', { evt_type: 'table:created' });

19
packages/nocodb/src/lib/noco/meta/helpers/apiMetrics.ts

@ -0,0 +1,19 @@
import { Request } from 'express';
import { Tele } from 'nc-help';
const countMap = {};
const metrics = async (req: Request) => {
if (!req?.route?.path) return;
const event = `a:api:${req.route.path}:${req.method}`;
countMap[event] = (countMap[event] || 0) + 1;
if (countMap[event] >= 50) {
Tele.event({ event });
countMap[event] = 0;
}
};
export default async (req: Request, _res, next) => {
metrics(req).then(() => {});
next();
};
Loading…
Cancel
Save