From 685b57fe59ee62510fa234b69277c47b634297f3 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 6 Oct 2022 19:57:01 +0530 Subject: [PATCH 01/13] feat(api): add metaCount method for getting records count Signed-off-by: Pranav C --- packages/nocodb/src/lib/meta/NcMetaIO.ts | 10 ++++++ packages/nocodb/src/lib/meta/NcMetaIOImpl.ts | 32 ++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/packages/nocodb/src/lib/meta/NcMetaIO.ts b/packages/nocodb/src/lib/meta/NcMetaIO.ts index fbcf0b6eb1..642920ef2b 100644 --- a/packages/nocodb/src/lib/meta/NcMetaIO.ts +++ b/packages/nocodb/src/lib/meta/NcMetaIO.ts @@ -162,6 +162,16 @@ export default abstract class NcMetaIO { } ): Promise; + public abstract metaCount( + project_id: string, + base_id: string, + target: string, + args?: { + condition?: { [key: string]: any }; + xcCondition?: XcCondition; + } + ): Promise; + public abstract metaPaginatedList( project_id: string, dbAlias: string, diff --git a/packages/nocodb/src/lib/meta/NcMetaIOImpl.ts b/packages/nocodb/src/lib/meta/NcMetaIOImpl.ts index d09953744b..271e3c20ea 100644 --- a/packages/nocodb/src/lib/meta/NcMetaIOImpl.ts +++ b/packages/nocodb/src/lib/meta/NcMetaIOImpl.ts @@ -366,6 +366,38 @@ export default class NcMetaIOImpl extends NcMetaIO { return query; } + + public async metaCount( + project_id: string, + dbAlias: string, + target: string, + args?: { + condition?: { [p: string]: any }; + xcCondition?; + } + ): Promise { + const query = this.knexConnection(target); + + if (project_id !== null && project_id !== undefined) { + query.where('project_id', project_id); + } + if (dbAlias !== null && dbAlias !== undefined) { + query.where('base_id', dbAlias); + } + + if (args?.condition) { + query.where(args.condition); + } + + if (args?.xcCondition) { + (query as any).condition(args.xcCondition); + } + + query.count('id', { as: 'count' }).first(); + + return +(await query)?.['count'] || 0; + } + public async metaUpdate( project_id: string, dbAlias: string, From 457b0b1128e960f3b0ba5f75633646808fd51925 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 6 Oct 2022 20:17:13 +0530 Subject: [PATCH 02/13] feat(api): add option to pass aggregate field name Signed-off-by: Pranav C --- packages/nocodb/src/lib/meta/NcMetaIO.ts | 1 + packages/nocodb/src/lib/meta/NcMetaIOImpl.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/meta/NcMetaIO.ts b/packages/nocodb/src/lib/meta/NcMetaIO.ts index 642920ef2b..70c5bf2444 100644 --- a/packages/nocodb/src/lib/meta/NcMetaIO.ts +++ b/packages/nocodb/src/lib/meta/NcMetaIO.ts @@ -169,6 +169,7 @@ export default abstract class NcMetaIO { args?: { condition?: { [key: string]: any }; xcCondition?: XcCondition; + aggField?: string; } ): Promise; diff --git a/packages/nocodb/src/lib/meta/NcMetaIOImpl.ts b/packages/nocodb/src/lib/meta/NcMetaIOImpl.ts index 271e3c20ea..ff265bc4de 100644 --- a/packages/nocodb/src/lib/meta/NcMetaIOImpl.ts +++ b/packages/nocodb/src/lib/meta/NcMetaIOImpl.ts @@ -374,6 +374,7 @@ export default class NcMetaIOImpl extends NcMetaIO { args?: { condition?: { [p: string]: any }; xcCondition?; + aggField?: string; } ): Promise { const query = this.knexConnection(target); @@ -393,7 +394,7 @@ export default class NcMetaIOImpl extends NcMetaIO { (query as any).condition(args.xcCondition); } - query.count('id', { as: 'count' }).first(); + query.count(args?.aggField || 'id', { as: 'count' }).first(); return +(await query)?.['count'] || 0; } From 5cc81095e6fe18eba61fbf0eedba162c51efb531 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 6 Oct 2022 23:41:08 +0530 Subject: [PATCH 03/13] feat(api): api for getting aggregated meta info Signed-off-by: Pranav C --- packages/nocodb/src/lib/meta/api/utilApis.ts | 169 ++++++++++++++++++- 1 file changed, 167 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/meta/api/utilApis.ts b/packages/nocodb/src/lib/meta/api/utilApis.ts index 7d5b462345..800c938954 100644 --- a/packages/nocodb/src/lib/meta/api/utilApis.ts +++ b/packages/nocodb/src/lib/meta/api/utilApis.ts @@ -2,6 +2,11 @@ import { Request, Response } from 'express'; import { packageVersion } from 'nc-help'; +import { ViewTypes } from 'nocodb-sdk'; +import Project from '../../models/Project'; +import Noco from '../../Noco'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { MetaTable } from '../../utils/globals'; import ncMetaAclMw from '../helpers/ncMetaAclMw'; import SqlMgrv2 from '../../db/sql-mgr/v2/SqlMgrv2'; import NcConfigFactory, { @@ -19,6 +24,7 @@ const versionCache = { export async function testConnection(req: Request, res: Response) { res.json(await SqlMgrv2.testConnection(req.body)); } + export async function appInfo(req: Request, res: Response) { const projectHasAdmin = !(await User.isFirst()); const result = { @@ -171,14 +177,172 @@ export async function axiosRequestMake(req: Request, res: Response) { export async function urlToDbConfig(req: Request, res: Response) { const { url } = req.body; try { - let connectionConfig; - connectionConfig = NcConfigFactory.extractXcUrlFromJdbc(url, true); + const connectionConfig = NcConfigFactory.extractXcUrlFromJdbc(url, true); return res.json(connectionConfig); } catch (error) { return res.sendStatus(500); } } +interface AllMeta { + projectCount: number; + projects: { + tableCount: { + table: number; + view: number; + }; + viewCount: { + formCount: number; + gridCount: number; + galleryCount: number; + kanbanCount: number; + total: number; + sharedFormCount: number; + sharedGridCount: number; + sharedGalleryCount: number; + sharedKanbanCount: number; + sharedTotal: number; + sharedPasswordProtected: number; + }; + webhookCount: number; + filterCount: number; + sortCount: number; + rowCount: { totalRecords: number }[]; + userCount: number; + }[]; + userCount: number; + sharedBaseCount: number; +} + +export async function allMeta(_req: Request, res: Response) { + const result: AllMeta = { + projectCount: 0, + projects: [], + userCount: await Noco.ncMeta.metaCount(null, null, MetaTable.USERS), + sharedBaseCount: 0, + }; + + const projects = await Project.list({}); + const userCount = await User.count(); + result.projectCount = projects.length; + result.userCount = userCount; + for (const project of projects) { + if (project.uuid) result.sharedBaseCount++; + const tableCount = await Noco.ncMeta.metaCount( + null, + null, + MetaTable.MODELS, + { + condition: { + project_id: project.id, + type: 'table', + }, + } + ); + + const dbViewCount = await Noco.ncMeta.metaCount( + null, + null, + MetaTable.MODELS, + { + condition: { + project_id: project.id, + type: 'view', + }, + } + ); + const views = await Noco.ncMeta.metaList2(null, null, MetaTable.VIEWS); + const viewCount = views.reduce( + (out, view) => { + out.total++; + + switch (view.type) { + case ViewTypes.GRID: + out.gridCount++; + if (view.uuid) out.sharedGridCount++; + break; + case ViewTypes.FORM: + out.formCount++; + if (view.uuid) out.sharedFormCount++; + break; + case ViewTypes.GALLERY: + out.galleryCount++; + if (view.uuid) out.sharedGalleryCount++; + break; + case ViewTypes.KANBAN: + out.kanbanCount++; + if (view.uuid) out.sharedKanbanCount++; + } + + if (view.uuid && view.password) out.sharedPasswordProtected++; + + return out; + }, + { + formCount: 0, + gridCount: 0, + galleryCount: 0, + kanbanCount: 0, + total: 0, + sharedFormCount: 0, + sharedGridCount: 0, + sharedGalleryCount: 0, + sharedKanbanCount: 0, + sharedTotal: 0, + sharedPasswordProtected: 0, + } + ); + + result.projects.push({ + tableCount: { table: tableCount, view: dbViewCount }, + + viewCount, + webhookCount: await Noco.ncMeta.metaCount(null, null, MetaTable.HOOKS, { + condition: { + project_id: project.id, + }, + }), + filterCount: await Noco.ncMeta.metaCount( + null, + null, + MetaTable.FILTER_EXP, + { + condition: { + project_id: project.id, + }, + } + ), + sortCount: await Noco.ncMeta.metaCount(null, null, MetaTable.SORT, { + condition: { + project_id: project.id, + }, + }), + rowCount: await project.getBases().then((bases) => { + return Promise.all( + bases.map((base) => + NcConnectionMgrv2.getSqlClient(base) + .totalRecords?.() + ?.then((result) => result?.data) + ) + ); + }), + userCount: await Noco.ncMeta.metaCount( + null, + null, + MetaTable.PROJECT_USERS, + { + condition: { + project_id: project.id, + }, + aggField: '*', + } + ), + }); + } + + res.json(result); +} + export default (router) => { router.post( '/api/v1/db/meta/connection/test', @@ -190,4 +354,5 @@ export default (router) => { router.get('/api/v1/health', catchError(appHealth)); router.get('/api/v1/feedback_form', catchError(feedbackFormGet)); router.post('/api/v1/url_to_config', catchError(urlToDbConfig)); + router.get('/api/v1/all_meta', catchError(allMeta)); }; From 6c868817fd9c636f6255989023f3f24d977f8be5 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 7 Oct 2022 00:25:15 +0530 Subject: [PATCH 04/13] test(api): add test for aggregated meta Signed-off-by: Pranav C --- packages/nocodb-sdk/package-lock.json | 283 +++++++++++++++++- .../tests/unit/rest/tests/project.test.ts | 50 +++- 2 files changed, 324 insertions(+), 9 deletions(-) diff --git a/packages/nocodb-sdk/package-lock.json b/packages/nocodb-sdk/package-lock.json index 407893fd5b..497bd8f603 100644 --- a/packages/nocodb-sdk/package-lock.json +++ b/packages/nocodb-sdk/package-lock.json @@ -538,6 +538,19 @@ "node": ">=8" } }, + "node_modules/nyc/node_modules/yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dev": true, + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/table/node_modules/ansi-styles": { "version": "4.3.0", "dev": true, @@ -1565,6 +1578,18 @@ "node": ">=10" } }, + "node_modules/gh-pages/node_modules/array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha512-Dxr6QJj/RdU/hCaBjOfxW+q6lyuVE6JFWIrAUpuOOhoJJoQ99cUn3igRaHVB5P9WrgFVN0FfArM3x0cueOU8ng==", + "dev": true, + "dependencies": { + "array-uniq": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/escape-goat": { "version": "2.1.1", "dev": true, @@ -1786,6 +1811,15 @@ "node": ">=0.10.0" } }, + "node_modules/nyc/node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/npm-run-all/node_modules/path-key": { "version": "2.0.1", "dev": true, @@ -2106,6 +2140,15 @@ "node": ">= 8" } }, + "node_modules/gh-pages/node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "dev": true, @@ -2539,6 +2582,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typedoc/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "node_modules/trim-repeated": { "version": "1.0.0", "dev": true, @@ -2970,6 +3022,18 @@ "node": ">=8" } }, + "node_modules/git-semver-tags/node_modules/type-fest": { + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.18.1.tgz", + "integrity": "sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/strip-indent": { "version": "3.0.0", "dev": true, @@ -4094,6 +4158,15 @@ "node": ">=0.8.0" } }, + "node_modules/inquirer/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/dotgitignore/node_modules/path-exists": { "version": "3.0.0", "dev": true, @@ -5144,6 +5217,15 @@ "once": "^1.4.0" } }, + "node_modules/array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha512-MNha4BWQ6JbwhFhj03YK552f7cb3AzoE8SzeljgChvL1dl3IcvggXVz1DilzySZkCja+CXuZbdW7yATchWn8/Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/escalade": { "version": "3.1.1", "dev": true, @@ -5160,6 +5242,21 @@ "node": ">=4.0.0" } }, + "node_modules/standard-version/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/eslint-module-utils/node_modules/p-limit": { "version": "1.3.0", "dev": true, @@ -5596,6 +5693,15 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/standard-version/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "dev": true, @@ -7101,6 +7207,18 @@ "node": ">=4.8" } }, + "node_modules/standard-version/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/spawn-wrap": { "version": "2.0.0", "dev": true, @@ -7133,6 +7251,18 @@ "dev": true, "license": "MIT" }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/strip-json-comments": { "version": "2.0.1", "dev": true, @@ -7210,6 +7340,21 @@ "license": "MIT", "optional": true }, + "node_modules/standard-version/node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@nodelib/fs.walk": { "version": "1.2.8", "dev": true, @@ -7463,6 +7608,21 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/standard-version/node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "dev": true, @@ -8451,6 +8611,15 @@ "ini": "^1.3.2" } }, + "node_modules/standard-version/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/semver-diff": { "version": "3.1.1", "dev": true, @@ -10468,6 +10637,12 @@ "version": "2.1.0", "dev": true }, + "array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha512-MNha4BWQ6JbwhFhj03YK552f7cb3AzoE8SzeljgChvL1dl3IcvggXVz1DilzySZkCja+CXuZbdW7yATchWn8/Q==", + "dev": true + }, "array.prototype.flat": { "version": "1.2.5", "dev": true, @@ -12614,6 +12789,15 @@ "globby": "^6.1.0" }, "dependencies": { + "array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha512-Dxr6QJj/RdU/hCaBjOfxW+q6lyuVE6JFWIrAUpuOOhoJJoQ99cUn3igRaHVB5P9WrgFVN0FfArM3x0cueOU8ng==", + "dev": true, + "requires": { + "array-uniq": "^1.0.1" + } + }, "commander": { "version": "2.20.3", "dev": true @@ -12641,6 +12825,12 @@ "pify": { "version": "2.3.0", "dev": true + }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true } } }, @@ -12737,6 +12927,12 @@ "semver": { "version": "6.3.0", "dev": true + }, + "type-fest": { + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.18.1.tgz", + "integrity": "sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==", + "dev": true } } }, @@ -13104,6 +13300,12 @@ "version": "3.0.0", "dev": true }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + }, "restore-cursor": { "version": "2.0.0", "dev": true, @@ -14064,6 +14266,12 @@ "color-convert": "^2.0.1" } }, + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true + }, "cliui": { "version": "6.0.0", "dev": true, @@ -14105,6 +14313,16 @@ "y18n": "^4.0.0", "yargs-parser": "^18.1.2" } + }, + "yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dev": true, + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } } } }, @@ -15042,6 +15260,12 @@ "supports-color": "^5.3.0" } }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true + }, "find-up": { "version": "5.0.0", "dev": true, @@ -15049,6 +15273,48 @@ "locate-path": "^6.0.0", "path-exists": "^4.0.0" } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true + }, + "locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "requires": { + "p-locate": "^5.0.0" + } + }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "requires": { + "p-limit": "^3.0.2" + } + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } } } }, @@ -15528,6 +15794,15 @@ "shiki": "^0.10.1" }, "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0" + } + }, "glob": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/glob/-/glob-8.0.3.tgz", @@ -15804,6 +16079,12 @@ "yn": { "version": "3.1.1", "dev": true + }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true } } -} +} \ No newline at end of file diff --git a/packages/nocodb/tests/unit/rest/tests/project.test.ts b/packages/nocodb/tests/unit/rest/tests/project.test.ts index caf563d8fc..c36a842fb0 100644 --- a/packages/nocodb/tests/unit/rest/tests/project.test.ts +++ b/packages/nocodb/tests/unit/rest/tests/project.test.ts @@ -1,10 +1,12 @@ -import 'mocha'; -import request from 'supertest'; -import init from '../../init/index'; -import { createProject, createSharedBase } from '../../factory/project'; -import { beforeEach } from 'mocha'; -import { Exception } from 'handlebars'; -import Project from '../../../../src/lib/models/Project'; +import 'mocha' +import request from 'supertest' +import { createTable } from '../../factory/table' +import init from '../../init/index' +import { createProject, createSharedBase } from '../../factory/project' +import { beforeEach } from 'mocha' +import { Exception } from 'handlebars' +import Project from '../../../../src/lib/models/Project' +import { expect } from 'chai' function projectTest() { let context; @@ -260,7 +262,39 @@ function projectTest() { .set('xc-auth', context.token) .send() .expect(200); - }); + }) + + + it.only('Get all projects meta', async () => { + await createTable(context, project, { table_name: 'table1', title: 'table1' }) + await createTable(context, project, { table_name: 'table2', title: 'table2' }) + await createTable(context, project, { table_name: 'table3', title: 'table3' }) + + await request(context.app) + .get(`/api/v1/all_meta`) + .set('xc-auth', context.token) + .send({}) + .expect(200) + .then(res => { + expect(res.body).to.have.all.keys('userCount', 'sharedBaseCount', 'projectCount','projects') + expect(res.body).to.have.property('projectCount').to.eq(1) + expect(res.body).to.have.property('projects').to.be.an('array') + expect(res.body.projects[0].tableCount.table).to.be.eq(3) + expect(res.body).to.have.nested.property('projects[0].tableCount.table').to.be.a('number') + expect(res.body).to.have.nested.property('projects[0].tableCount.view').to.be.a('number') + expect(res.body).to.have.nested.property('projects[0].viewCount').to.be.an('object').have.keys('formCount', 'gridCount', 'galleryCount', 'kanbanCount', 'total', 'sharedFormCount', 'sharedGridCount', 'sharedGalleryCount', 'sharedKanbanCount', 'sharedTotal', 'sharedPasswordProtected') + expect(res.body.projects[0]).have.keys( + 'webhookCount', + 'filterCount', + 'sortCount', + 'userCount', + 'rowCount', + 'tableCount', + 'viewCount' + ) + expect(res.body).to.have.nested.property('projects[0].rowCount').to.be.an('array') + }) + }) } export default function () { From 9074012b67e3f02e729f37d125f9aae70d3c433a Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 7 Oct 2022 12:04:30 +0530 Subject: [PATCH 05/13] refactor(api): update api path Signed-off-by: Pranav C --- packages/nocodb/src/lib/meta/api/utilApis.ts | 7 +- .../tests/unit/rest/tests/project.test.ts | 140 +++++++++--------- 2 files changed, 75 insertions(+), 72 deletions(-) diff --git a/packages/nocodb/src/lib/meta/api/utilApis.ts b/packages/nocodb/src/lib/meta/api/utilApis.ts index 800c938954..dcbf73a561 100644 --- a/packages/nocodb/src/lib/meta/api/utilApis.ts +++ b/packages/nocodb/src/lib/meta/api/utilApis.ts @@ -214,7 +214,7 @@ interface AllMeta { sharedBaseCount: number; } -export async function allMeta(_req: Request, res: Response) { +export async function getAggregatedMetaInfo(_req: Request, res: Response) { const result: AllMeta = { projectCount: 0, projects: [], @@ -354,5 +354,8 @@ export default (router) => { router.get('/api/v1/health', catchError(appHealth)); router.get('/api/v1/feedback_form', catchError(feedbackFormGet)); router.post('/api/v1/url_to_config', catchError(urlToDbConfig)); - router.get('/api/v1/all_meta', catchError(allMeta)); + router.get( + '/api/v1/aggregated-meta-info', + ncMetaAclMw(getAggregatedMetaInfo, 'getAggregatedMetaInfo') + ); }; diff --git a/packages/nocodb/tests/unit/rest/tests/project.test.ts b/packages/nocodb/tests/unit/rest/tests/project.test.ts index c36a842fb0..0881d328a6 100644 --- a/packages/nocodb/tests/unit/rest/tests/project.test.ts +++ b/packages/nocodb/tests/unit/rest/tests/project.test.ts @@ -9,22 +9,22 @@ import Project from '../../../../src/lib/models/Project' import { expect } from 'chai' function projectTest() { - let context; - let project; + let context + let project - beforeEach(async function () { - context = await init(); + beforeEach(async function() { + context = await init() - project = await createProject(context); - }); + project = await createProject(context) + }) it('Get project info', async () => { await request(context.app) .get(`/api/v1/db/meta/projects/${project.id}/info`) .set('xc-auth', context.token) .send({}) - .expect(200); - }); + .expect(200) + }) // todo: Test by creating models under project and check if the UCL is working it('UI ACL', async () => { @@ -32,8 +32,8 @@ function projectTest() { .get(`/api/v1/db/meta/projects/${project.id}/visibility-rules`) .set('xc-auth', context.token) .send({}) - .expect(200); - }); + .expect(200) + }) // todo: Test creating visibility set it('List projects', async () => { @@ -41,11 +41,11 @@ function projectTest() { .get('/api/v1/db/meta/projects/') .set('xc-auth', context.token) .send({}) - .expect(200); + .expect(200) - if (response.body.list.length !== 1) new Error('Should list only 1 project'); - if (!response.body.pageInfo) new Error('Should have pagination info'); - }); + if (response.body.list.length !== 1) new Error('Should list only 1 project') + if (!response.body.pageInfo) new Error('Should have pagination info') + }) it('Create project', async () => { const response = await request(context.app) @@ -54,11 +54,11 @@ function projectTest() { .send({ title: 'Title1', }) - .expect(200); + .expect(200) - const newProject = await Project.getByTitleOrId(response.body.id); - if (!newProject) return new Error('Project not created'); - }); + const newProject = await Project.getByTitleOrId(response.body.id) + if (!newProject) return new Error('Project not created') + }) it('Create projects with existing title', async () => { await request(context.app) @@ -67,8 +67,8 @@ function projectTest() { .send({ title: project.title, }) - .expect(400); - }); + .expect(400) + }) // todo: fix passport user role popluation bug // it('Delete project', async async () => { @@ -83,7 +83,7 @@ function projectTest() { // }) // .expect(200, async (err) => { // // console.log(res); - // + // // const deletedProject = await Project.getByTitleOrId( // toBeDeletedProject.id @@ -99,10 +99,10 @@ function projectTest() { .get(`/api/v1/db/meta/projects/${project.id}`) .set('xc-auth', context.token) .send() - .expect(200); + .expect(200) - if (response.body.id !== project.id) return new Error('Got the wrong project'); - }); + if (response.body.id !== project.id) return new Error('Got the wrong project') + }) it('Update projects', async () => { await request(context.app) @@ -111,18 +111,18 @@ function projectTest() { .send({ title: 'NewTitle', }) - .expect(200); + .expect(200) - const newProject = await Project.getByTitleOrId(project.id); - if (newProject.title !== 'NewTitle') { - return new Error('Project not updated'); - } - }); + const newProject = await Project.getByTitleOrId(project.id) + if (newProject.title !== 'NewTitle') { + return new Error('Project not updated') + } + }) - it('Update projects with existing title', async function () { + it('Update projects with existing title', async function() { const newProject = await createProject(context, { title: 'NewTitle1', - }); + }) await request(context.app) .patch(`/api/v1/db/meta/projects/${project.id}`) @@ -130,8 +130,8 @@ function projectTest() { .send({ title: newProject.title, }) - .expect(400); - }); + .expect(400) + }) it('Create project shared base', async () => { await request(context.app) @@ -141,18 +141,18 @@ function projectTest() { roles: 'viewer', password: 'test', }) - .expect(200); + .expect(200) - const updatedProject = await Project.getByTitleOrId(project.id); + const updatedProject = await Project.getByTitleOrId(project.id) if ( !updatedProject.uuid || updatedProject.roles !== 'viewer' || updatedProject.password !== 'test' ) { - return new Error('Shared base not configured properly'); + return new Error('Shared base not configured properly') } - }); + }) it('Created project shared base should have only editor or viewer role', async () => { await request(context.app) @@ -162,17 +162,17 @@ function projectTest() { roles: 'commenter', password: 'test', }) - .expect(200); + .expect(200) - const updatedProject = await Project.getByTitleOrId(project.id); + const updatedProject = await Project.getByTitleOrId(project.id) if (updatedProject.roles === 'commenter') { - return new Error('Shared base not configured properly'); + return new Error('Shared base not configured properly') } - }); + }) it('Updated project shared base should have only editor or viewer role', async () => { - await createSharedBase(context.app, context.token, project); + await createSharedBase(context.app, context.token, project) await request(context.app) .patch(`/api/v1/db/meta/projects/${project.id}/shared`) @@ -181,17 +181,17 @@ function projectTest() { roles: 'commenter', password: 'test', }) - .expect(200); + .expect(200) - const updatedProject = await Project.getByTitleOrId(project.id); + const updatedProject = await Project.getByTitleOrId(project.id) if (updatedProject.roles === 'commenter') { - throw new Exception('Shared base not updated properly'); + throw new Exception('Shared base not updated properly') } - }); + }) it('Updated project shared base', async () => { - await createSharedBase(context.app, context.token, project); + await createSharedBase(context.app, context.token, project) await request(context.app) .patch(`/api/v1/db/meta/projects/${project.id}/shared`) @@ -200,42 +200,42 @@ function projectTest() { roles: 'editor', password: 'test', }) - .expect(200); - const updatedProject = await Project.getByTitleOrId(project.id); + .expect(200) + const updatedProject = await Project.getByTitleOrId(project.id) if (updatedProject.roles !== 'editor') { - throw new Exception('Shared base not updated properly'); + throw new Exception('Shared base not updated properly') } - }); + }) it('Get project shared base', async () => { - await createSharedBase(context.app, context.token, project); + await createSharedBase(context.app, context.token, project) await request(context.app) .get(`/api/v1/db/meta/projects/${project.id}/shared`) .set('xc-auth', context.token) .send() - .expect(200); + .expect(200) - const updatedProject = await Project.getByTitleOrId(project.id); + const updatedProject = await Project.getByTitleOrId(project.id) if (!updatedProject.uuid) { - throw new Exception('Shared base not created'); + throw new Exception('Shared base not created') } - }); + }) it('Delete project shared base', async () => { - await createSharedBase(context.app, context.token, project); + await createSharedBase(context.app, context.token, project) await request(context.app) .delete(`/api/v1/db/meta/projects/${project.id}/shared`) .set('xc-auth', context.token) .send() - .expect(200); - const updatedProject = await Project.getByTitleOrId(project.id); + .expect(200) + const updatedProject = await Project.getByTitleOrId(project.id) if (updatedProject.uuid) { - throw new Exception('Shared base not deleted'); + throw new Exception('Shared base not deleted') } - }); + }) // todo: Do compare api test @@ -244,16 +244,16 @@ function projectTest() { .get(`/api/v1/db/meta/projects/${project.id}/meta-diff`) .set('xc-auth', context.token) .send() - .expect(200); - }); + .expect(200) + }) it('Meta diff sync', async () => { await request(context.app) .post(`/api/v1/db/meta/projects/${project.id}/meta-diff`) .set('xc-auth', context.token) .send() - .expect(200); - }); + .expect(200) + }) // todo: improve test. Check whether the all the actions are present in the response and correct as well it('Meta diff sync', async () => { @@ -265,13 +265,13 @@ function projectTest() { }) - it.only('Get all projects meta', async () => { + it('Get all projects meta', async () => { await createTable(context, project, { table_name: 'table1', title: 'table1' }) await createTable(context, project, { table_name: 'table2', title: 'table2' }) await createTable(context, project, { table_name: 'table3', title: 'table3' }) await request(context.app) - .get(`/api/v1/all_meta`) + .get(`/api/v1/aggregated-meta-info`) .set('xc-auth', context.token) .send({}) .expect(200) @@ -297,6 +297,6 @@ function projectTest() { }) } -export default function () { - describe('Project', projectTest); +export default function() { + describe('Project', projectTest) } From 15ee45922d722a95db05e69a5bf988cbfb5a84fd Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 7 Oct 2022 12:17:07 +0530 Subject: [PATCH 06/13] feat(swagger): add api in swagger Signed-off-by: Pranav C --- packages/nocodb-sdk/src/lib/Api.ts | 16 +++ scripts/sdk/swagger.json | 159 +++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) diff --git a/packages/nocodb-sdk/src/lib/Api.ts b/packages/nocodb-sdk/src/lib/Api.ts index e3c4e531c9..24a2f60528 100644 --- a/packages/nocodb-sdk/src/lib/Api.ts +++ b/packages/nocodb-sdk/src/lib/Api.ts @@ -3301,6 +3301,22 @@ export class Api< ...params, }), + /** + * No description + * + * @tags Utils + * @name AggregatedMetaInfo + * @request GET:/api/v1/aggregated-meta-info + * @response `200` `any` OK + */ + aggregatedMetaInfo: (params: RequestParams = {}) => + this.request({ + path: `/api/v1/aggregated-meta-info`, + method: 'GET', + format: 'json', + ...params, + }), + /** * @description Get All K/V pairs in NocoCache * diff --git a/scripts/sdk/swagger.json b/scripts/sdk/swagger.json index 0e639602ef..31c890e036 100644 --- a/scripts/sdk/swagger.json +++ b/scripts/sdk/swagger.json @@ -5337,6 +5337,165 @@ "description": "" } }, + "/api/v1/aggregated-meta-info": { + "parameters": [], + "get": { + "summary": "", + "operationId": "utils-aggregated-meta-info", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": {} + }, + "application/xml": { + "schema": { + "type": "object", + "properties": { + "projectCount": { + "type": "integer" + }, + "projects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "tableCount": { + "type": "object", + "properties": { + "table": { + "type": "integer" + }, + "view": { + "type": "integer" + } + } + }, + "viewCount": { + "type": "object", + "properties": { + "formCount": { + "type": "integer" + }, + "gridCount": { + "type": "integer" + }, + "galleryCount": { + "type": "integer" + }, + "kanbanCount": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "sharedFormCount": { + "type": "integer" + }, + "sharedGridCount": { + "type": "integer" + }, + "sharedGalleryCount": { + "type": "integer" + }, + "sharedKanbanCount": { + "type": "integer" + }, + "sharedTotal": { + "type": "integer" + }, + "sharedPasswordProtected": { + "type": "integer" + } + } + }, + "webhookCount": { + "type": "integer" + }, + "filterCount": { + "type": "integer" + }, + "sortCount": { + "type": "integer" + }, + "rowCount": { + "type": "array", + "items": { + "type": "object", + "properties": { + "TotalRecords": { + "type": "string" + } + } + } + }, + "userCount": { + "type": "integer" + } + } + } + }, + "userCount": { + "type": "integer" + }, + "sharedBaseCount": { + "type": "integer" + } + }, + "x-examples": { + "Example 1": { + "projectCount": 1, + "projects": [ + { + "tableCount": { + "table": 3, + "view": 0 + }, + "viewCount": { + "formCount": 0, + "gridCount": 3, + "galleryCount": 0, + "kanbanCount": 0, + "total": 3, + "sharedFormCount": 0, + "sharedGridCount": 0, + "sharedGalleryCount": 0, + "sharedKanbanCount": 0, + "sharedTotal": 0, + "sharedPasswordProtected": 0 + }, + "webhookCount": 0, + "filterCount": 0, + "sortCount": 0, + "rowCount": [ + { + "TotalRecords": "76" + } + ], + "userCount": 1 + } + ], + "userCount": 1, + "sharedBaseCount": 0 + } + } + }, + "examples": { + "example-1": { + "value": "{\n \"projectCount\": 1,\n \"projects\": [\n {\n \"tableCount\": {\n \"table\": 3,\n \"view\": 0\n },\n \"viewCount\": {\n \"formCount\": 0,\n \"gridCount\": 3,\n \"galleryCount\": 0,\n \"kanbanCount\": 0,\n \"total\": 3,\n \"sharedFormCount\": 0,\n \"sharedGridCount\": 0,\n \"sharedGalleryCount\": 0,\n \"sharedKanbanCount\": 0,\n \"sharedTotal\": 0,\n \"sharedPasswordProtected\": 0\n },\n \"webhookCount\": 0,\n \"filterCount\": 0,\n \"sortCount\": 0,\n \"rowCount\": [\n {\n \"TotalRecords\": \"76\"\n }\n ],\n \"userCount\": 1\n }\n ],\n \"userCount\": 1,\n \"sharedBaseCount\": 0\n}" + } + } + } + } + } + }, + "tags": [ + "Utils" + ], + "description": "" + } + }, "/api/v1/db/meta/cache": { "get": { "summary": "Your GET endpoint", From c356bb1ec3ba1624f2a31292891d5634e2ce7dfb Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 7 Oct 2022 14:50:57 +0530 Subject: [PATCH 07/13] refactor(api): do all db operations parallel Signed-off-by: Pranav C --- packages/nocodb/src/lib/meta/api/utilApis.ts | 248 ++++++++++--------- 1 file changed, 132 insertions(+), 116 deletions(-) diff --git a/packages/nocodb/src/lib/meta/api/utilApis.ts b/packages/nocodb/src/lib/meta/api/utilApis.ts index dcbf73a561..121c6bad94 100644 --- a/packages/nocodb/src/lib/meta/api/utilApis.ts +++ b/packages/nocodb/src/lib/meta/api/utilApis.ts @@ -187,6 +187,7 @@ export async function urlToDbConfig(req: Request, res: Response) { interface AllMeta { projectCount: number; projects: { + external?: boolean; tableCount: { table: number; view: number; @@ -202,7 +203,7 @@ interface AllMeta { sharedGalleryCount: number; sharedKanbanCount: number; sharedTotal: number; - sharedPasswordProtected: number; + sharedLockedCount: number; }; webhookCount: number; filterCount: number; @@ -215,130 +216,145 @@ interface AllMeta { } export async function getAggregatedMetaInfo(_req: Request, res: Response) { + const [projects, userCount] = await Promise.all([ + Project.list({}), + Noco.ncMeta.metaCount(null, null, MetaTable.USERS), + ]); + const result: AllMeta = { - projectCount: 0, + projectCount: projects.length, projects: [], - userCount: await Noco.ncMeta.metaCount(null, null, MetaTable.USERS), + userCount, sharedBaseCount: 0, }; - const projects = await Project.list({}); - const userCount = await User.count(); - result.projectCount = projects.length; - result.userCount = userCount; - for (const project of projects) { - if (project.uuid) result.sharedBaseCount++; - const tableCount = await Noco.ncMeta.metaCount( - null, - null, - MetaTable.MODELS, - { - condition: { - project_id: project.id, - type: 'table', - }, - } - ); - - const dbViewCount = await Noco.ncMeta.metaCount( - null, - null, - MetaTable.MODELS, - { - condition: { - project_id: project.id, - type: 'view', - }, - } - ); - const views = await Noco.ncMeta.metaList2(null, null, MetaTable.VIEWS); - const viewCount = views.reduce( - (out, view) => { - out.total++; - - switch (view.type) { - case ViewTypes.GRID: - out.gridCount++; - if (view.uuid) out.sharedGridCount++; - break; - case ViewTypes.FORM: - out.formCount++; - if (view.uuid) out.sharedFormCount++; - break; - case ViewTypes.GALLERY: - out.galleryCount++; - if (view.uuid) out.sharedGalleryCount++; - break; - case ViewTypes.KANBAN: - out.kanbanCount++; - if (view.uuid) out.sharedKanbanCount++; - } + result.projects.push( + ...(await Promise.all( + projects.map(async (project) => { + if (project.uuid) result.sharedBaseCount++; + const [ + tableCount, + dbViewCount, + viewCount, + webhookCount, + filterCount, + sortCount, + rowCount, + userCount, + ] = await Promise.all([ + // db tables count + Noco.ncMeta.metaCount(null, null, MetaTable.MODELS, { + condition: { + project_id: project.id, + type: 'table', + }, + }), + // db views count + Noco.ncMeta.metaCount(null, null, MetaTable.MODELS, { + condition: { + project_id: project.id, + type: 'view', + }, + }), + // views count + (async () => { + const views = await Noco.ncMeta.metaList2( + null, + null, + MetaTable.VIEWS + ); + // grid, form, gallery, kanban and shared count + return views.reduce( + (out, view) => { + out.total++; - if (view.uuid && view.password) out.sharedPasswordProtected++; + switch (view.type) { + case ViewTypes.GRID: + out.gridCount++; + if (view.uuid) out.sharedGridCount++; + break; + case ViewTypes.FORM: + out.formCount++; + if (view.uuid) out.sharedFormCount++; + break; + case ViewTypes.GALLERY: + out.galleryCount++; + if (view.uuid) out.sharedGalleryCount++; + break; + case ViewTypes.KANBAN: + out.kanbanCount++; + if (view.uuid) out.sharedKanbanCount++; + } - return out; - }, - { - formCount: 0, - gridCount: 0, - galleryCount: 0, - kanbanCount: 0, - total: 0, - sharedFormCount: 0, - sharedGridCount: 0, - sharedGalleryCount: 0, - sharedKanbanCount: 0, - sharedTotal: 0, - sharedPasswordProtected: 0, - } - ); + if (view.uuid && view.password) out.sharedLockedCount++; - result.projects.push({ - tableCount: { table: tableCount, view: dbViewCount }, + return out; + }, + { + formCount: 0, + gridCount: 0, + galleryCount: 0, + kanbanCount: 0, + total: 0, + sharedFormCount: 0, + sharedGridCount: 0, + sharedGalleryCount: 0, + sharedKanbanCount: 0, + sharedTotal: 0, + sharedLockedCount: 0, + } + ); + })(), + // webhooks count + Noco.ncMeta.metaCount(null, null, MetaTable.HOOKS, { + condition: { + project_id: project.id, + }, + }), + // filters count + Noco.ncMeta.metaCount(null, null, MetaTable.FILTER_EXP, { + condition: { + project_id: project.id, + }, + }), + // sorts count + Noco.ncMeta.metaCount(null, null, MetaTable.SORT, { + condition: { + project_id: project.id, + }, + }), + // row count per base + project.getBases().then((bases) => { + return Promise.all( + bases.map((base) => + NcConnectionMgrv2.getSqlClient(base) + .totalRecords?.() + ?.then((result) => result?.data) + ) + ); + }), + // project users count + Noco.ncMeta.metaCount(null, null, MetaTable.PROJECT_USERS, { + condition: { + project_id: project.id, + }, + aggField: '*', + }), + ]); - viewCount, - webhookCount: await Noco.ncMeta.metaCount(null, null, MetaTable.HOOKS, { - condition: { - project_id: project.id, - }, - }), - filterCount: await Noco.ncMeta.metaCount( - null, - null, - MetaTable.FILTER_EXP, - { - condition: { - project_id: project.id, - }, - } - ), - sortCount: await Noco.ncMeta.metaCount(null, null, MetaTable.SORT, { - condition: { - project_id: project.id, - }, - }), - rowCount: await project.getBases().then((bases) => { - return Promise.all( - bases.map((base) => - NcConnectionMgrv2.getSqlClient(base) - .totalRecords?.() - ?.then((result) => result?.data) - ) - ); - }), - userCount: await Noco.ncMeta.metaCount( - null, - null, - MetaTable.PROJECT_USERS, - { - condition: { - project_id: project.id, - }, - aggField: '*', - } - ), - }); - } + return { + tableCount: { table: tableCount, view: dbViewCount }, + external: !project.is_meta, + viewCount, + webhookCount, + filterCount, + sortCount, + rowCount, + userCount, + }; + }) + )) + ); res.json(result); } From 4adc242f7ea700b2f3ef48bd58a9369e7b6a7268 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 7 Oct 2022 16:03:08 +0530 Subject: [PATCH 08/13] refactor(api): handle all error cases and on error log and return as null Signed-off-by: Pranav C --- packages/nocodb/src/lib/meta/api/utilApis.ts | 299 ++++++++++--------- 1 file changed, 153 insertions(+), 146 deletions(-) diff --git a/packages/nocodb/src/lib/meta/api/utilApis.ts b/packages/nocodb/src/lib/meta/api/utilApis.ts index 121c6bad94..c490fc55c8 100644 --- a/packages/nocodb/src/lib/meta/api/utilApis.ts +++ b/packages/nocodb/src/lib/meta/api/utilApis.ts @@ -184,33 +184,38 @@ export async function urlToDbConfig(req: Request, res: Response) { } } +interface ViewCount { + formCount: number | null; + gridCount: number | null; + galleryCount: number | null; + kanbanCount: number | null; + total: number | null; + sharedFormCount: number | null; + sharedGridCount: number | null; + sharedGalleryCount: number | null; + sharedKanbanCount: number | null; + sharedTotal: number | null; + sharedLockedCount: number | null; +} + interface AllMeta { projectCount: number; - projects: { - external?: boolean; - tableCount: { - table: number; - view: number; - }; - viewCount: { - formCount: number; - gridCount: number; - galleryCount: number; - kanbanCount: number; - total: number; - sharedFormCount: number; - sharedGridCount: number; - sharedGalleryCount: number; - sharedKanbanCount: number; - sharedTotal: number; - sharedLockedCount: number; - }; - webhookCount: number; - filterCount: number; - sortCount: number; - rowCount: { totalRecords: number }[]; - userCount: number; - }[]; + projects: ( + | { + external?: boolean | null; + tableCount: { + table: number; + view: number; + } | null; + viewCount: ViewCount; + webhookCount: number | null; + filterCount: number | null; + sortCount: number | null; + rowCount: ({ totalRecords: number } | null)[] | null; + userCount: number | null; + } + | { error: string } + )[]; userCount: number; sharedBaseCount: number; } @@ -229,136 +234,138 @@ export async function getAggregatedMetaInfo(_req: Request, res: Response) { }; result.projects.push( - ...(await Promise.all( - projects.map(async (project) => { - if (project.uuid) result.sharedBaseCount++; - const [ - tableCount, - dbViewCount, - viewCount, - webhookCount, - filterCount, - sortCount, - rowCount, - userCount, - ] = await Promise.all([ - // db tables count - Noco.ncMeta.metaCount(null, null, MetaTable.MODELS, { - condition: { - project_id: project.id, - type: 'table', - }, - }), - // db views count - Noco.ncMeta.metaCount(null, null, MetaTable.MODELS, { - condition: { - project_id: project.id, - type: 'view', - }, - }), - // views count - (async () => { - const views = await Noco.ncMeta.metaList2( - null, - null, - MetaTable.VIEWS - ); - // grid, form, gallery, kanban and shared count - return views.reduce( - (out, view) => { - out.total++; + ...extractResultOrNull( + await Promise.allSettled( + projects.map(async (project) => { + if (project.uuid) result.sharedBaseCount++; + const [ + tableCount, + dbViewCount, + viewCount, + webhookCount, + filterCount, + sortCount, + rowCount, + userCount, + ] = extractResultOrNull( + await Promise.allSettled([ + // db tables count + Noco.ncMeta.metaCount(project.id, null, MetaTable.MODELS, { + condition: { + type: 'table', + }, + }), + // db views count + Noco.ncMeta.metaCount(project.id, null, MetaTable.MODELS, { + condition: { + type: 'view', + }, + }), + // views count + (async () => { + const views = await Noco.ncMeta.metaList2( + project.id, + null, + MetaTable.VIEWS + ); + // grid, form, gallery, kanban and shared count + return views.reduce( + (out, view) => { + out.total++; - switch (view.type) { - case ViewTypes.GRID: - out.gridCount++; - if (view.uuid) out.sharedGridCount++; - break; - case ViewTypes.FORM: - out.formCount++; - if (view.uuid) out.sharedFormCount++; - break; - case ViewTypes.GALLERY: - out.galleryCount++; - if (view.uuid) out.sharedGalleryCount++; - break; - case ViewTypes.KANBAN: - out.kanbanCount++; - if (view.uuid) out.sharedKanbanCount++; - } + switch (view.type) { + case ViewTypes.GRID: + out.gridCount++; + if (view.uuid) out.sharedGridCount++; + break; + case ViewTypes.FORM: + out.formCount++; + if (view.uuid) out.sharedFormCount++; + break; + case ViewTypes.GALLERY: + out.galleryCount++; + if (view.uuid) out.sharedGalleryCount++; + break; + case ViewTypes.KANBAN: + out.kanbanCount++; + if (view.uuid) out.sharedKanbanCount++; + } - if (view.uuid && view.password) out.sharedLockedCount++; + if (view.uuid && view.password) out.sharedLockedCount++; - return out; - }, - { - formCount: 0, - gridCount: 0, - galleryCount: 0, - kanbanCount: 0, - total: 0, - sharedFormCount: 0, - sharedGridCount: 0, - sharedGalleryCount: 0, - sharedKanbanCount: 0, - sharedTotal: 0, - sharedLockedCount: 0, - } - ); - })(), - // webhooks count - Noco.ncMeta.metaCount(null, null, MetaTable.HOOKS, { - condition: { - project_id: project.id, - }, - }), - // filters count - Noco.ncMeta.metaCount(null, null, MetaTable.FILTER_EXP, { - condition: { - project_id: project.id, - }, - }), - // sorts count - Noco.ncMeta.metaCount(null, null, MetaTable.SORT, { - condition: { - project_id: project.id, - }, - }), - // row count per base - project.getBases().then((bases) => { - return Promise.all( - bases.map((base) => - NcConnectionMgrv2.getSqlClient(base) - .totalRecords?.() - ?.then((result) => result?.data) - ) - ); - }), - // project users count - Noco.ncMeta.metaCount(null, null, MetaTable.PROJECT_USERS, { - condition: { - project_id: project.id, - }, - aggField: '*', - }), - ]); + return out; + }, + { + formCount: 0, + gridCount: 0, + galleryCount: 0, + kanbanCount: 0, + total: 0, + sharedFormCount: 0, + sharedGridCount: 0, + sharedGalleryCount: 0, + sharedKanbanCount: 0, + sharedTotal: 0, + sharedLockedCount: 0, + } + ); + })(), + // webhooks count + Noco.ncMeta.metaCount(project.id, null, MetaTable.HOOKS), + // filters count + Noco.ncMeta.metaCount(project.id, null, MetaTable.FILTER_EXP), + // sorts count + Noco.ncMeta.metaCount(project.id, null, MetaTable.SORT), + // row count per base + project.getBases().then(async (bases) => { + return extractResultOrNull( + await Promise.allSettled( + bases.map((base) => + NcConnectionMgrv2.getSqlClient(base) + .totalRecords?.() + ?.then((result) => result?.data) + ) + ) + ); + }), + // project users count + Noco.ncMeta.metaCount(null, null, MetaTable.PROJECT_USERS, { + condition: { + project_id: project.id, + }, + aggField: '*', + }), + ]) + ); - return { - tableCount: { table: tableCount, view: dbViewCount }, - external: !project.is_meta, - viewCount, - webhookCount, - filterCount, - sortCount, - rowCount, - userCount, - }; - }) - )) + return { + tableCount: { table: tableCount.status, view: dbViewCount }, + external: !project.is_meta, + viewCount, + webhookCount, + filterCount, + sortCount, + rowCount, + userCount, + }; + }) + ) + ) ); res.json(result); } +const extractResultOrNull = (results: PromiseSettledResult[]) => { + return results.map((result) => { + if (result.status === 'fulfilled') { + return result.value; + } + console.log(result.reason); + return null; + }); +}; + export default (router) => { router.post( '/api/v1/db/meta/connection/test', From e0029317c5f42e2adb32d86fd0f5726190445344 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 7 Oct 2022 16:03:42 +0530 Subject: [PATCH 09/13] feat(gui): add copy option Signed-off-by: Pranav C --- packages/nc-gui/pages/index/index/index.vue | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/nc-gui/pages/index/index/index.vue b/packages/nc-gui/pages/index/index/index.vue index a4fd7fe006..ea0fb6097d 100644 --- a/packages/nc-gui/pages/index/index/index.vue +++ b/packages/nc-gui/pages/index/index/index.vue @@ -16,6 +16,7 @@ import { themeV2Colors, useApi, useBreakpoints, + useCopy, useGlobal, useNuxtApp, useUIPermission, @@ -130,12 +131,20 @@ const customRow = (record: ProjectType) => ({ }) onBeforeMount(loadProjects) + +const { copy } = useCopy() + +const copyProjectMeta = async () => { + const aggregatedMetaInfo = await $api.utils.aggregatedMetaInfo() + copy(JSON.stringify(aggregatedMetaInfo)) + message.info('Copied aggregated project meta to clipboard') +}