From 00c942f98c0873894c79bed746ca24e103e2f5d2 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 23:47:26 +0530 Subject: [PATCH 01/64] refactor: move api handlers to controllers folder Signed-off-by: Pranav C --- packages/nocodb/src/lib/Noco.ts | 2 +- .../{meta/api => controllers}/apiTokenApis.ts | 10 +-- .../api => controllers}/attachmentApis.ts | 18 ++--- .../{meta/api => controllers}/auditApis.ts | 10 +-- .../lib/{meta/api => controllers}/baseApis.ts | 14 ++-- .../{meta/api => controllers}/cacheApis.ts | 4 +- .../{meta/api => controllers}/columnApis.ts | 44 +++++------ .../dataApis/bulkDataAliasApis.ts | 12 +-- .../dataApis/dataAliasApis.ts | 20 ++--- .../dataApis/dataAliasExportApis.ts | 6 +- .../dataApis/dataAliasNestedApis.ts | 14 ++-- .../api => controllers}/dataApis/dataApis.ts | 18 ++--- .../api => controllers}/dataApis/helpers.ts | 20 ++--- .../api => controllers}/dataApis/index.ts | 0 .../dataApis/oldDataApis.ts | 18 ++--- .../{meta/api => controllers}/exportApis.ts | 4 +- .../{meta/api => controllers}/filterApis.ts | 16 ++-- .../{meta/api => controllers}/formViewApis.ts | 18 ++--- .../api => controllers}/formViewColumnApis.ts | 8 +- .../api => controllers}/galleryViewApis.ts | 10 +-- .../{meta/api => controllers}/gridViewApis.ts | 18 ++--- .../api => controllers}/gridViewColumnApis.ts | 8 +- .../lib/{meta/api => controllers}/hookApis.ts | 18 ++--- .../api => controllers}/hookFilterApis.ts | 16 ++-- .../api => controllers}/kanbanViewApis.ts | 10 +-- .../{meta/api => controllers}/mapViewApis.ts | 8 +- .../{meta/api => controllers}/metaDiffApis.ts | 28 +++---- .../modelVisibilityApis.ts | 10 +-- .../api => controllers}/orgLicenseApis.ts | 12 +-- .../{meta/api => controllers}/orgTokenApis.ts | 16 ++-- .../{meta/api => controllers}/orgUserApis.ts | 30 +++---- .../{meta/api => controllers}/pluginApis.ts | 12 +-- .../{meta/api => controllers}/projectApis.ts | 28 +++---- .../api => controllers}/projectUserApis.ts | 26 +++---- .../api => controllers}/publicApis/index.ts | 0 .../publicApis/publicDataApis.ts | 24 +++--- .../publicApis/publicDataExportApis.ts | 18 ++--- .../publicApis/publicMetaApis.ts | 14 ++-- .../api => controllers}/sharedBaseApis.ts | 8 +- .../lib/{meta/api => controllers}/sortApis.ts | 16 ++-- .../swagger/helpers/getPaths.ts | 6 +- .../swagger/helpers/getSchemas.ts | 6 +- .../swagger/helpers/getSwaggerColumnMetas.ts | 10 +-- .../swagger/helpers/getSwaggerJSON.ts | 14 ++-- .../swagger/helpers/swagger-base.json | 0 .../swagger/helpers/templates/headers.ts | 0 .../swagger/helpers/templates/params.ts | 2 +- .../swagger/helpers/templates/paths.ts | 0 .../swagger/helpers/templates/schemas.ts | 0 .../api => controllers}/swagger/redocHtml.ts | 0 .../swagger/swaggerApis.ts | 8 +- .../swagger/swaggerHtml.ts | 0 .../sync/helpers/EntityMap.ts | 0 .../sync/helpers/NocoSyncDestAdapter.ts | 0 .../sync/helpers/NocoSyncSourceAdapter.ts | 0 .../sync/helpers/fetchAT.ts | 0 .../api => controllers}/sync/helpers/job.ts | 0 .../sync/helpers/readAndProcessData.ts | 0 .../sync/helpers/syncMap.ts | 0 .../api => controllers}/sync/importApis.ts | 8 +- .../sync/syncSourceApis.ts | 8 +- .../{meta/api => controllers}/tableApis.ts | 34 ++++---- .../lib/{meta/api => controllers}/testApis.ts | 2 +- .../api => controllers}/userApi/helpers.ts | 4 +- .../api => controllers}/userApi/index.ts | 0 .../userApi/initAdminFromEnv.ts | 10 +-- .../userApi/initStrategies.ts | 16 ++-- .../userApi/ui/auth/emailVerify.ts | 0 .../userApi/ui/auth/resetPassword.ts | 0 .../ui/emailTemplates/forgotPassword.ts | 0 .../userApi/ui/emailTemplates/invite.ts | 0 .../userApi/ui/emailTemplates/verify.ts | 0 .../api => controllers}/userApi/userApis.ts | 24 +++--- .../lib/{meta/api => controllers}/utilApis.ts | 22 +++--- .../lib/{meta/api => controllers}/viewApis.ts | 14 ++-- .../api => controllers}/viewColumnApis.ts | 6 +- .../src/lib/meta/api/helpers/populateMeta.ts | 2 +- packages/nocodb/src/lib/meta/api/index.ts | 78 +++++++++---------- 78 files changed, 430 insertions(+), 430 deletions(-) rename packages/nocodb/src/lib/{meta/api => controllers}/apiTokenApis.ts (84%) rename packages/nocodb/src/lib/{meta/api => controllers}/attachmentApis.ts (91%) rename packages/nocodb/src/lib/{meta/api => controllers}/auditApis.ts (91%) rename packages/nocodb/src/lib/{meta/api => controllers}/baseApis.ts (87%) rename packages/nocodb/src/lib/{meta/api => controllers}/cacheApis.ts (84%) rename packages/nocodb/src/lib/{meta/api => controllers}/columnApis.ts (97%) rename packages/nocodb/src/lib/{meta/api => controllers}/dataApis/bulkDataAliasApis.ts (88%) rename packages/nocodb/src/lib/{meta/api => controllers}/dataApis/dataAliasApis.ts (95%) rename packages/nocodb/src/lib/{meta/api => controllers}/dataApis/dataAliasExportApis.ts (93%) rename packages/nocodb/src/lib/{meta/api => controllers}/dataApis/dataAliasNestedApis.ts (94%) rename packages/nocodb/src/lib/{meta/api => controllers}/dataApis/dataApis.ts (96%) rename packages/nocodb/src/lib/{meta/api => controllers}/dataApis/helpers.ts (91%) rename packages/nocodb/src/lib/{meta/api => controllers}/dataApis/index.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/dataApis/oldDataApis.ts (89%) rename packages/nocodb/src/lib/{meta/api => controllers}/exportApis.ts (88%) rename packages/nocodb/src/lib/{meta/api => controllers}/filterApis.ts (89%) rename packages/nocodb/src/lib/{meta/api => controllers}/formViewApis.ts (78%) rename packages/nocodb/src/lib/{meta/api => controllers}/formViewColumnApis.ts (71%) rename packages/nocodb/src/lib/{meta/api => controllers}/galleryViewApis.ts (84%) rename packages/nocodb/src/lib/{meta/api => controllers}/gridViewApis.ts (69%) rename packages/nocodb/src/lib/{meta/api => controllers}/gridViewColumnApis.ts (78%) rename packages/nocodb/src/lib/{meta/api => controllers}/hookApis.ts (84%) rename packages/nocodb/src/lib/{meta/api => controllers}/hookFilterApis.ts (87%) rename packages/nocodb/src/lib/{meta/api => controllers}/kanbanViewApis.ts (84%) rename packages/nocodb/src/lib/{meta/api => controllers}/mapViewApis.ts (86%) rename packages/nocodb/src/lib/{meta/api => controllers}/metaDiffApis.ts (97%) rename packages/nocodb/src/lib/{meta/api => controllers}/modelVisibilityApis.ts (92%) rename packages/nocodb/src/lib/{meta/api => controllers}/orgLicenseApis.ts (75%) rename packages/nocodb/src/lib/{meta/api => controllers}/orgTokenApis.ts (81%) rename packages/nocodb/src/lib/{meta/api => controllers}/orgUserApis.ts (91%) rename packages/nocodb/src/lib/{meta/api => controllers}/pluginApis.ts (83%) rename packages/nocodb/src/lib/{meta/api => controllers}/projectApis.ts (90%) rename packages/nocodb/src/lib/{meta/api => controllers}/projectUserApis.ts (92%) rename packages/nocodb/src/lib/{meta/api => controllers}/publicApis/index.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/publicApis/publicDataApis.ts (94%) rename packages/nocodb/src/lib/{meta/api => controllers}/publicApis/publicDataExportApis.ts (92%) rename packages/nocodb/src/lib/{meta/api => controllers}/publicApis/publicMetaApis.ts (88%) rename packages/nocodb/src/lib/{meta/api => controllers}/sharedBaseApis.ts (93%) rename packages/nocodb/src/lib/{meta/api => controllers}/sortApis.ts (82%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/helpers/getPaths.ts (88%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/helpers/getSchemas.ts (88%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/helpers/getSwaggerColumnMetas.ts (83%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/helpers/getSwaggerJSON.ts (80%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/helpers/swagger-base.json (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/helpers/templates/headers.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/helpers/templates/params.ts (98%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/helpers/templates/paths.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/helpers/templates/schemas.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/redocHtml.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/swaggerApis.ts (86%) rename packages/nocodb/src/lib/{meta/api => controllers}/swagger/swaggerHtml.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/sync/helpers/EntityMap.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/sync/helpers/NocoSyncDestAdapter.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/sync/helpers/NocoSyncSourceAdapter.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/sync/helpers/fetchAT.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/sync/helpers/job.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/sync/helpers/readAndProcessData.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/sync/helpers/syncMap.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/sync/importApis.ts (94%) rename packages/nocodb/src/lib/{meta/api => controllers}/sync/syncSourceApis.ts (89%) rename packages/nocodb/src/lib/{meta/api => controllers}/tableApis.ts (92%) rename packages/nocodb/src/lib/{meta/api => controllers}/testApis.ts (85%) rename packages/nocodb/src/lib/{meta/api => controllers}/userApi/helpers.ts (83%) rename packages/nocodb/src/lib/{meta/api => controllers}/userApi/index.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/userApi/initAdminFromEnv.ts (97%) rename packages/nocodb/src/lib/{meta/api => controllers}/userApi/initStrategies.ts (96%) rename packages/nocodb/src/lib/{meta/api => controllers}/userApi/ui/auth/emailVerify.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/userApi/ui/auth/resetPassword.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/userApi/ui/emailTemplates/forgotPassword.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/userApi/ui/emailTemplates/invite.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/userApi/ui/emailTemplates/verify.ts (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/userApi/userApis.ts (96%) rename packages/nocodb/src/lib/{meta/api => controllers}/utilApis.ts (95%) rename packages/nocodb/src/lib/{meta/api => controllers}/viewApis.ts (91%) rename packages/nocodb/src/lib/{meta/api => controllers}/viewColumnApis.ts (88%) diff --git a/packages/nocodb/src/lib/Noco.ts b/packages/nocodb/src/lib/Noco.ts index 9099de28b4..00b80d421b 100644 --- a/packages/nocodb/src/lib/Noco.ts +++ b/packages/nocodb/src/lib/Noco.ts @@ -45,7 +45,7 @@ import User from './models/User'; import * as http from 'http'; import weAreHiring from './utils/weAreHiring'; import getInstance from './utils/getInstance'; -import initAdminFromEnv from './meta/api/userApi/initAdminFromEnv'; +import initAdminFromEnv from './controllers/userApi/initAdminFromEnv'; const log = debug('nc:app'); require('dotenv').config(); diff --git a/packages/nocodb/src/lib/meta/api/apiTokenApis.ts b/packages/nocodb/src/lib/controllers/apiTokenApis.ts similarity index 84% rename from packages/nocodb/src/lib/meta/api/apiTokenApis.ts rename to packages/nocodb/src/lib/controllers/apiTokenApis.ts index 5ed4c08094..33e940c0e5 100644 --- a/packages/nocodb/src/lib/meta/api/apiTokenApis.ts +++ b/packages/nocodb/src/lib/controllers/apiTokenApis.ts @@ -1,11 +1,11 @@ import { Request, Response, Router } from 'express'; import { OrgUserRoles } from 'nocodb-sdk'; import { Tele } from 'nc-help'; -import { NcError } from '../helpers/catchError'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import ApiToken from '../../models/ApiToken'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import { NcError } from '../meta/helpers/catchError'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import ApiToken from '../models/ApiToken'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; export async function apiTokenList(req: Request, res: Response) { res.json(await ApiToken.list(req['user'].id)); diff --git a/packages/nocodb/src/lib/meta/api/attachmentApis.ts b/packages/nocodb/src/lib/controllers/attachmentApis.ts similarity index 91% rename from packages/nocodb/src/lib/meta/api/attachmentApis.ts rename to packages/nocodb/src/lib/controllers/attachmentApis.ts index 88097286b4..2255197ff8 100644 --- a/packages/nocodb/src/lib/meta/api/attachmentApis.ts +++ b/packages/nocodb/src/lib/controllers/attachmentApis.ts @@ -5,16 +5,16 @@ import { nanoid } from 'nanoid'; import { OrgUserRoles, ProjectRoles } from 'nocodb-sdk'; import path from 'path'; import slash from 'slash'; -import Noco from '../../Noco'; -import { MetaTable } from '../../utils/globals'; -import mimetypes, { mimeIcons } from '../../utils/mimeTypes'; +import Noco from '../Noco'; +import { MetaTable } from '../utils/globals'; +import mimetypes, { mimeIcons } from '../utils/mimeTypes'; import { Tele } from 'nc-help'; -import extractProjectIdAndAuthenticate from '../helpers/extractProjectIdAndAuthenticate'; -import catchError, { NcError } from '../helpers/catchError'; -import NcPluginMgrv2 from '../helpers/NcPluginMgrv2'; -import Local from '../../v1-legacy/plugins/adapters/storage/Local'; -import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants'; -import { getCacheMiddleware } from './helpers'; +import extractProjectIdAndAuthenticate from '../meta/helpers/extractProjectIdAndAuthenticate'; +import catchError, { NcError } from '../meta/helpers/catchError'; +import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; +import Local from '../v1-legacy/plugins/adapters/storage/Local'; +import { NC_ATTACHMENT_FIELD_SIZE } from '../constants'; +import { getCacheMiddleware } from '../meta/api/helpers'; const isUploadAllowed = async (req: Request, _res: Response, next: any) => { if (!req['user']?.id) { diff --git a/packages/nocodb/src/lib/meta/api/auditApis.ts b/packages/nocodb/src/lib/controllers/auditApis.ts similarity index 91% rename from packages/nocodb/src/lib/meta/api/auditApis.ts rename to packages/nocodb/src/lib/controllers/auditApis.ts index fdf9f70daf..96b0acddee 100644 --- a/packages/nocodb/src/lib/meta/api/auditApis.ts +++ b/packages/nocodb/src/lib/controllers/auditApis.ts @@ -1,12 +1,12 @@ import { Request, Response, Router } from 'express'; -import Audit from '../../models/Audit'; +import Audit from '../models/Audit'; import { AuditOperationSubTypes, AuditOperationTypes } from 'nocodb-sdk'; -import Model from '../../models/Model'; -import { PagedResponseImpl } from '../helpers/PagedResponse'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; +import Model from '../models/Model'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import DOMPurify from 'isomorphic-dompurify'; -import { getAjvValidatorMw } from './helpers'; +import { getAjvValidatorMw } from '../meta/api/helpers'; export async function commentRow(req: Request, res) { res.json( diff --git a/packages/nocodb/src/lib/meta/api/baseApis.ts b/packages/nocodb/src/lib/controllers/baseApis.ts similarity index 87% rename from packages/nocodb/src/lib/meta/api/baseApis.ts rename to packages/nocodb/src/lib/controllers/baseApis.ts index c13bbd18d3..1ac4d3e3ba 100644 --- a/packages/nocodb/src/lib/meta/api/baseApis.ts +++ b/packages/nocodb/src/lib/controllers/baseApis.ts @@ -1,13 +1,13 @@ import { Request, Response } from 'express'; -import Project from '../../models/Project'; +import Project from '../models/Project'; import { BaseListType } from 'nocodb-sdk'; -import { PagedResponseImpl } from '../helpers/PagedResponse'; -import { syncBaseMigration } from '../helpers/syncMigration'; -import Base from '../../models/Base'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import { syncBaseMigration } from '../meta/helpers/syncMigration'; +import Base from '../models/Base'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { Tele } from 'nc-help'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw, populateMeta } from './helpers'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw, populateMeta } from '../meta/api/helpers'; export async function baseGet( req: Request, diff --git a/packages/nocodb/src/lib/meta/api/cacheApis.ts b/packages/nocodb/src/lib/controllers/cacheApis.ts similarity index 84% rename from packages/nocodb/src/lib/meta/api/cacheApis.ts rename to packages/nocodb/src/lib/controllers/cacheApis.ts index 8c5af7fa0d..9e1fab6777 100644 --- a/packages/nocodb/src/lib/meta/api/cacheApis.ts +++ b/packages/nocodb/src/lib/controllers/cacheApis.ts @@ -1,5 +1,5 @@ -import catchError from '../helpers/catchError'; -import NocoCache from '../../cache/NocoCache'; +import catchError from '../meta/helpers/catchError'; +import NocoCache from '../cache/NocoCache'; import { Router } from 'express'; export async function cacheGet(_, res) { diff --git a/packages/nocodb/src/lib/meta/api/columnApis.ts b/packages/nocodb/src/lib/controllers/columnApis.ts similarity index 97% rename from packages/nocodb/src/lib/meta/api/columnApis.ts rename to packages/nocodb/src/lib/controllers/columnApis.ts index a5e9e8e9ec..78aa196dcb 100644 --- a/packages/nocodb/src/lib/meta/api/columnApis.ts +++ b/packages/nocodb/src/lib/controllers/columnApis.ts @@ -1,16 +1,16 @@ import { Request, Response, Router } from 'express'; -import Model from '../../models/Model'; -import ProjectMgrv2 from '../../db/sql-mgr/v2/ProjectMgrv2'; -import Base from '../../models/Base'; -import Column from '../../models/Column'; +import Model from '../models/Model'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; +import Base from '../models/Base'; +import Column from '../models/Column'; import { Tele } from 'nc-help'; -import validateParams from '../helpers/validateParams'; +import validateParams from '../meta/helpers/validateParams'; -import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; +import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; import { getUniqueColumnAliasName, getUniqueColumnName, -} from '../helpers/getUniqueName'; +} from '../meta/helpers/getUniqueName'; import { AuditOperationSubTypes, AuditOperationTypes, @@ -24,20 +24,20 @@ import { TableType, UITypes, } from 'nocodb-sdk'; -import Audit from '../../models/Audit'; -import SqlMgrv2 from '../../db/sql-mgr/v2/SqlMgrv2'; -import Noco from '../../Noco'; -import NcMetaIO from '../NcMetaIO'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { NcError } from '../helpers/catchError'; -import getColumnPropsFromUIDT from '../helpers/getColumnPropsFromUIDT'; -import mapDefaultDisplayValue from '../helpers/mapDefaultDisplayValue'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import FormulaColumn from '../../models/FormulaColumn'; -import KanbanView from '../../models/KanbanView'; -import { MetaTable } from '../../utils/globals'; -import formulaQueryBuilderv2 from '../../db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2'; +import Audit from '../models/Audit'; +import SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2'; +import Noco from '../Noco'; +import NcMetaIO from '../meta/NcMetaIO'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { NcError } from '../meta/helpers/catchError'; +import getColumnPropsFromUIDT from '../meta/helpers/getColumnPropsFromUIDT'; +import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import FormulaColumn from '../models/FormulaColumn'; +import KanbanView from '../models/KanbanView'; +import { MetaTable } from '../utils/globals'; +import formulaQueryBuilderv2 from '../db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2'; import { createHmAndBtColumn, generateFkName, @@ -46,7 +46,7 @@ import { validateLookupPayload, validateRequiredField, validateRollupPayload, -} from './helpers'; +} from '../meta/api/helpers'; export enum Altered { NEW_COLUMN = 1, diff --git a/packages/nocodb/src/lib/meta/api/dataApis/bulkDataAliasApis.ts b/packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasApis.ts similarity index 88% rename from packages/nocodb/src/lib/meta/api/dataApis/bulkDataAliasApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasApis.ts index 739585a1c5..bef5276be6 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/bulkDataAliasApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasApis.ts @@ -1,11 +1,11 @@ import { Request, Response, Router } from 'express'; -import { BaseModelSqlv2 } from '../../../db/sql-data-mapper/lib/sql/BaseModelSqlv2'; -import Model from '../../../models/Model'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import { BaseModelSqlv2 } from '../../db/sql-data-mapper/lib/sql/BaseModelSqlv2'; +import Model from '../../models/Model'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; import { getViewAndModelFromRequestByAliasOrId } from './helpers'; -import apiMetrics from '../../helpers/apiMetrics'; +import apiMetrics from '../../meta/helpers/apiMetrics'; type BulkOperation = | 'bulkInsert' diff --git a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasApis.ts similarity index 95% rename from packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/dataAliasApis.ts index 4ef4b7ce5e..a1bd604261 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataAliasApis.ts @@ -1,16 +1,16 @@ import { Request, Response, Router } from 'express'; -import Model from '../../../models/Model'; +import Model from '../../models/Model'; import { nocoExecute } from 'nc-help'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; -import { NcError } from '../../helpers/catchError'; -import { PagedResponseImpl } from '../../helpers/PagedResponse'; -import View from '../../../models/View'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { NcError } from '../../meta/helpers/catchError'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import View from '../../models/View'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; import { getViewAndModelFromRequestByAliasOrId } from './helpers'; -import apiMetrics from '../../helpers/apiMetrics'; -import getAst from '../../../db/sql-data-mapper/lib/sql/helpers/getAst'; -import { parseHrtimeToSeconds } from '../helpers'; +import apiMetrics from '../../meta/helpers/apiMetrics'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import { parseHrtimeToSeconds } from '../../meta/api/helpers'; // todo: Handle the error case where view doesnt belong to model async function dataList(req: Request, res: Response) { diff --git a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasExportApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts similarity index 93% rename from packages/nocodb/src/lib/meta/api/dataApis/dataAliasExportApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts index 8c6cf38bea..8593ff6fa9 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasExportApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts @@ -1,13 +1,13 @@ import { Request, Response, Router } from 'express'; import * as XLSX from 'xlsx'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; import { extractCsvData, extractXlsxData, getViewAndModelFromRequestByAliasOrId, } from './helpers'; -import apiMetrics from '../../helpers/apiMetrics'; -import View from '../../../models/View'; +import apiMetrics from '../../meta/helpers/apiMetrics'; +import View from '../../models/View'; async function excelDataExport(req: Request, res: Response) { const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); diff --git a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasNestedApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedApis.ts similarity index 94% rename from packages/nocodb/src/lib/meta/api/dataApis/dataAliasNestedApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedApis.ts index d260694fe5..5151af03b7 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasNestedApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedApis.ts @@ -1,15 +1,15 @@ import { Request, Response, Router } from 'express'; -import Model from '../../../models/Model'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; -import { PagedResponseImpl } from '../../helpers/PagedResponse'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import Model from '../../models/Model'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; import { getColumnByIdOrName, getViewAndModelFromRequestByAliasOrId, } from './helpers'; -import { NcError } from '../../helpers/catchError'; -import apiMetrics from '../../helpers/apiMetrics'; +import { NcError } from '../../meta/helpers/catchError'; +import apiMetrics from '../../meta/helpers/apiMetrics'; // todo: handle case where the given column is not ltar export async function mmList(req: Request, res: Response, next) { diff --git a/packages/nocodb/src/lib/meta/api/dataApis/dataApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataApis.ts similarity index 96% rename from packages/nocodb/src/lib/meta/api/dataApis/dataApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/dataApis.ts index b8a8899b5f..7c25607aab 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/dataApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataApis.ts @@ -1,14 +1,14 @@ import { Request, Response, Router } from 'express'; -import Model from '../../../models/Model'; +import Model from '../../models/Model'; import { nocoExecute } from 'nc-help'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; -import { PagedResponseImpl } from '../../helpers/PagedResponse'; -import View from '../../../models/View'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import { NcError } from '../../helpers/catchError'; -import apiMetrics from '../../helpers/apiMetrics'; -import getAst from '../../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import View from '../../models/View'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import { NcError } from '../../meta/helpers/catchError'; +import apiMetrics from '../../meta/helpers/apiMetrics'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; export async function dataList(req: Request, res: Response, next) { const view = await View.get(req.params.viewId); diff --git a/packages/nocodb/src/lib/meta/api/dataApis/helpers.ts b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts similarity index 91% rename from packages/nocodb/src/lib/meta/api/dataApis/helpers.ts rename to packages/nocodb/src/lib/controllers/dataApis/helpers.ts index aec0e4216e..a2b94737f2 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/helpers.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts @@ -1,20 +1,20 @@ -import Project from '../../../models/Project'; -import Model from '../../../models/Model'; -import View from '../../../models/View'; -import { NcError } from '../../helpers/catchError'; +import Project from '../../models/Project'; +import Model from '../../models/Model'; +import View from '../../models/View'; +import { NcError } from '../../meta/helpers/catchError'; import { Request } from 'express'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; import { isSystemColumn, UITypes } from 'nocodb-sdk'; import { nocoExecute } from 'nc-help'; import * as XLSX from 'xlsx'; -import Column from '../../../models/Column'; -import LookupColumn from '../../../models/LookupColumn'; -import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; +import Column from '../../models/Column'; +import LookupColumn from '../../models/LookupColumn'; +import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; import papaparse from 'papaparse'; -import getAst from '../../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; export async function getViewAndModelFromRequestByAliasOrId( req: | Request<{ projectName: string; tableName: string; viewName?: string }> diff --git a/packages/nocodb/src/lib/meta/api/dataApis/index.ts b/packages/nocodb/src/lib/controllers/dataApis/index.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/dataApis/index.ts rename to packages/nocodb/src/lib/controllers/dataApis/index.ts diff --git a/packages/nocodb/src/lib/meta/api/dataApis/oldDataApis.ts b/packages/nocodb/src/lib/controllers/dataApis/oldDataApis.ts similarity index 89% rename from packages/nocodb/src/lib/meta/api/dataApis/oldDataApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/oldDataApis.ts index 5df008eb8a..3492ff7361 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/oldDataApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/oldDataApis.ts @@ -1,14 +1,14 @@ import { Request, Response, Router } from 'express'; -import Model from '../../../models/Model'; +import Model from '../../models/Model'; import { nocoExecute } from 'nc-help'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; -import View from '../../../models/View'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import Project from '../../../models/Project'; -import { NcError } from '../../helpers/catchError'; -import apiMetrics from '../../helpers/apiMetrics'; -import getAst from '../../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import View from '../../models/View'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import Project from '../../models/Project'; +import { NcError } from '../../meta/helpers/catchError'; +import apiMetrics from '../../meta/helpers/apiMetrics'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; export async function dataList(req: Request, res: Response) { const { model, view } = await getViewAndModelFromRequest(req); diff --git a/packages/nocodb/src/lib/meta/api/exportApis.ts b/packages/nocodb/src/lib/controllers/exportApis.ts similarity index 88% rename from packages/nocodb/src/lib/meta/api/exportApis.ts rename to packages/nocodb/src/lib/controllers/exportApis.ts index 7c0a5e013e..1c043acff1 100644 --- a/packages/nocodb/src/lib/meta/api/exportApis.ts +++ b/packages/nocodb/src/lib/controllers/exportApis.ts @@ -1,6 +1,6 @@ import { Request, Response, Router } from 'express'; -import View from '../../models/View'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; +import View from '../models/View'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { extractCsvData } from './dataApis/helpers'; async function exportCsv(req: Request, res: Response) { diff --git a/packages/nocodb/src/lib/meta/api/filterApis.ts b/packages/nocodb/src/lib/controllers/filterApis.ts similarity index 89% rename from packages/nocodb/src/lib/meta/api/filterApis.ts rename to packages/nocodb/src/lib/controllers/filterApis.ts index 1be850b7d8..45b18b0104 100644 --- a/packages/nocodb/src/lib/meta/api/filterApis.ts +++ b/packages/nocodb/src/lib/controllers/filterApis.ts @@ -1,19 +1,19 @@ import { Request, Response, Router } from 'express'; // @ts-ignore -import Model from '../../models/Model'; +import Model from '../models/Model'; import { Tele } from 'nc-help'; // @ts-ignore -import { PagedResponseImpl } from '../helpers/PagedResponse'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; // @ts-ignore import { Table, TableList, TableListParams, TableReq } from 'nocodb-sdk'; // @ts-ignore -import ProjectMgrv2 from '../../db/sql-mgr/v2/ProjectMgrv2'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; // @ts-ignore -import Project from '../../models/Project'; -import Filter from '../../models/Filter'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import Project from '../models/Project'; +import Filter from '../models/Filter'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; // @ts-ignore export async function filterGet(req: Request, res: Response, next) { diff --git a/packages/nocodb/src/lib/meta/api/formViewApis.ts b/packages/nocodb/src/lib/controllers/formViewApis.ts similarity index 78% rename from packages/nocodb/src/lib/meta/api/formViewApis.ts rename to packages/nocodb/src/lib/controllers/formViewApis.ts index beadf3cded..02f8bc8e44 100644 --- a/packages/nocodb/src/lib/meta/api/formViewApis.ts +++ b/packages/nocodb/src/lib/controllers/formViewApis.ts @@ -1,19 +1,19 @@ import { Request, Response, Router } from 'express'; // @ts-ignore -import Model from '../../models/Model'; +import Model from '../models/Model'; import { Tele } from 'nc-help'; // @ts-ignore -import { PagedResponseImpl } from '../helpers/PagedResponse'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import { FormType, ViewTypes } from 'nocodb-sdk'; // @ts-ignore -import ProjectMgrv2 from '../../db/sql-mgr/v2/ProjectMgrv2'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; // @ts-ignore -import Project from '../../models/Project'; -import View from '../../models/View'; -import FormView from '../../models/FormView'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import Project from '../models/Project'; +import View from '../models/View'; +import FormView from '../models/FormView'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; // @ts-ignore export async function formViewGet(req: Request, res: Response) { diff --git a/packages/nocodb/src/lib/meta/api/formViewColumnApis.ts b/packages/nocodb/src/lib/controllers/formViewColumnApis.ts similarity index 71% rename from packages/nocodb/src/lib/meta/api/formViewColumnApis.ts rename to packages/nocodb/src/lib/controllers/formViewColumnApis.ts index bad2f91b0a..fd1b5ab8e7 100644 --- a/packages/nocodb/src/lib/meta/api/formViewColumnApis.ts +++ b/packages/nocodb/src/lib/controllers/formViewColumnApis.ts @@ -1,9 +1,9 @@ import { Request, Response, Router } from 'express'; -import FormViewColumn from '../../models/FormViewColumn'; +import FormViewColumn from '../models/FormViewColumn'; import { Tele } from 'nc-help'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; export async function columnUpdate(req: Request, res: Response) { Tele.emit('evt', { evt_type: 'formViewColumn:updated' }); diff --git a/packages/nocodb/src/lib/meta/api/galleryViewApis.ts b/packages/nocodb/src/lib/controllers/galleryViewApis.ts similarity index 84% rename from packages/nocodb/src/lib/meta/api/galleryViewApis.ts rename to packages/nocodb/src/lib/controllers/galleryViewApis.ts index 2c6c79db9e..d682f70fad 100644 --- a/packages/nocodb/src/lib/meta/api/galleryViewApis.ts +++ b/packages/nocodb/src/lib/controllers/galleryViewApis.ts @@ -1,11 +1,11 @@ import { Request, Response, Router } from 'express'; import { GalleryType, ViewTypes } from 'nocodb-sdk'; -import View from '../../models/View'; -import GalleryView from '../../models/GalleryView'; +import View from '../models/View'; +import GalleryView from '../models/GalleryView'; import { Tele } from 'nc-help'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; export async function galleryViewGet(req: Request, res: Response) { res.json(await GalleryView.get(req.params.galleryViewId)); } diff --git a/packages/nocodb/src/lib/meta/api/gridViewApis.ts b/packages/nocodb/src/lib/controllers/gridViewApis.ts similarity index 69% rename from packages/nocodb/src/lib/meta/api/gridViewApis.ts rename to packages/nocodb/src/lib/controllers/gridViewApis.ts index 96880398cb..2038717232 100644 --- a/packages/nocodb/src/lib/meta/api/gridViewApis.ts +++ b/packages/nocodb/src/lib/controllers/gridViewApis.ts @@ -1,19 +1,19 @@ import { Request, Router } from 'express'; // @ts-ignore -import Model from '../../models/Model'; +import Model from '../models/Model'; import { Tele } from 'nc-help'; // @ts-ignore -import { PagedResponseImpl } from '../helpers/PagedResponse'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import { ViewTypes } from 'nocodb-sdk'; // @ts-ignore -import ProjectMgrv2 from '../../db/sql-mgr/v2/ProjectMgrv2'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; // @ts-ignore -import Project from '../../models/Project'; -import View from '../../models/View'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import GridView from '../../models/GridView'; -import { getAjvValidatorMw } from './helpers'; +import Project from '../models/Project'; +import View from '../models/View'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import GridView from '../models/GridView'; +import { getAjvValidatorMw } from '../meta/api/helpers'; // @ts-ignore export async function gridViewCreate(req: Request, res) { diff --git a/packages/nocodb/src/lib/meta/api/gridViewColumnApis.ts b/packages/nocodb/src/lib/controllers/gridViewColumnApis.ts similarity index 78% rename from packages/nocodb/src/lib/meta/api/gridViewColumnApis.ts rename to packages/nocodb/src/lib/controllers/gridViewColumnApis.ts index 28669a6a2f..f18a175c9e 100644 --- a/packages/nocodb/src/lib/meta/api/gridViewColumnApis.ts +++ b/packages/nocodb/src/lib/controllers/gridViewColumnApis.ts @@ -1,9 +1,9 @@ import { Request, Response, Router } from 'express'; -import GridViewColumn from '../../models/GridViewColumn'; +import GridViewColumn from '../models/GridViewColumn'; import { Tele } from 'nc-help'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; export async function columnList(req: Request, res: Response) { res.json(await GridViewColumn.list(req.params.gridViewId)); diff --git a/packages/nocodb/src/lib/meta/api/hookApis.ts b/packages/nocodb/src/lib/controllers/hookApis.ts similarity index 84% rename from packages/nocodb/src/lib/meta/api/hookApis.ts rename to packages/nocodb/src/lib/controllers/hookApis.ts index bf0b29b6ec..1215b4604a 100644 --- a/packages/nocodb/src/lib/meta/api/hookApis.ts +++ b/packages/nocodb/src/lib/controllers/hookApis.ts @@ -1,15 +1,15 @@ import { Tele } from 'nc-help'; -import catchError from '../helpers/catchError'; +import catchError from '../meta/helpers/catchError'; import { Request, Response, Router } from 'express'; -import Hook from '../../models/Hook'; +import Hook from '../models/Hook'; import { HookListType, HookType } from 'nocodb-sdk'; -import { PagedResponseImpl } from '../helpers/PagedResponse'; -import { invokeWebhook } from '../helpers/webhookHelpers'; -import Model from '../../models/Model'; -import populateSamplePayload from '../helpers/populateSamplePayload'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import { invokeWebhook } from '../meta/helpers/webhookHelpers'; +import Model from '../models/Model'; +import populateSamplePayload from '../meta/helpers/populateSamplePayload'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; export async function hookList( req: Request, diff --git a/packages/nocodb/src/lib/meta/api/hookFilterApis.ts b/packages/nocodb/src/lib/controllers/hookFilterApis.ts similarity index 87% rename from packages/nocodb/src/lib/meta/api/hookFilterApis.ts rename to packages/nocodb/src/lib/controllers/hookFilterApis.ts index 7b182fcab4..0d0aa0a8da 100644 --- a/packages/nocodb/src/lib/meta/api/hookFilterApis.ts +++ b/packages/nocodb/src/lib/controllers/hookFilterApis.ts @@ -1,19 +1,19 @@ import { Request, Response, Router } from 'express'; // @ts-ignore -import Model from '../../models/Model'; +import Model from '../models/Model'; import { Tele } from 'nc-help'; // @ts-ignore -import { PagedResponseImpl } from '../helpers/PagedResponse'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; // @ts-ignore import { Table, TableList, TableListParams, TableReq } from 'nocodb-sdk'; // @ts-ignore -import ProjectMgrv2 from '../../db/sql-mgr/v2/ProjectMgrv2'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; // @ts-ignore -import Project from '../../models/Project'; -import Filter from '../../models/Filter'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import Project from '../models/Project'; +import Filter from '../models/Filter'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; // @ts-ignore export async function filterGet(req: Request, res: Response, next) { diff --git a/packages/nocodb/src/lib/meta/api/kanbanViewApis.ts b/packages/nocodb/src/lib/controllers/kanbanViewApis.ts similarity index 84% rename from packages/nocodb/src/lib/meta/api/kanbanViewApis.ts rename to packages/nocodb/src/lib/controllers/kanbanViewApis.ts index c8d864bf82..e9db2a3afb 100644 --- a/packages/nocodb/src/lib/meta/api/kanbanViewApis.ts +++ b/packages/nocodb/src/lib/controllers/kanbanViewApis.ts @@ -1,11 +1,11 @@ import { Request, Response, Router } from 'express'; import { KanbanType, ViewTypes } from 'nocodb-sdk'; -import View from '../../models/View'; -import KanbanView from '../../models/KanbanView'; +import View from '../models/View'; +import KanbanView from '../models/KanbanView'; import { Tele } from 'nc-help'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; export async function kanbanViewGet(req: Request, res: Response) { res.json(await KanbanView.get(req.params.kanbanViewId)); diff --git a/packages/nocodb/src/lib/meta/api/mapViewApis.ts b/packages/nocodb/src/lib/controllers/mapViewApis.ts similarity index 86% rename from packages/nocodb/src/lib/meta/api/mapViewApis.ts rename to packages/nocodb/src/lib/controllers/mapViewApis.ts index 5771284da4..9c5123f98a 100644 --- a/packages/nocodb/src/lib/meta/api/mapViewApis.ts +++ b/packages/nocodb/src/lib/controllers/mapViewApis.ts @@ -1,10 +1,10 @@ import { Request, Response, Router } from 'express'; import { MapType, ViewTypes } from 'nocodb-sdk'; -import View from '../../models/View'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; +import View from '../models/View'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { Tele } from 'nc-help'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import MapView from '../../models/MapView'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import MapView from '../models/MapView'; export async function mapViewGet(req: Request, res: Response) { res.json(await MapView.get(req.params.mapViewId)); diff --git a/packages/nocodb/src/lib/meta/api/metaDiffApis.ts b/packages/nocodb/src/lib/controllers/metaDiffApis.ts similarity index 97% rename from packages/nocodb/src/lib/meta/api/metaDiffApis.ts rename to packages/nocodb/src/lib/controllers/metaDiffApis.ts index 35f8118e21..551846074d 100644 --- a/packages/nocodb/src/lib/meta/api/metaDiffApis.ts +++ b/packages/nocodb/src/lib/controllers/metaDiffApis.ts @@ -1,22 +1,22 @@ // // Project CRUD import { Tele } from 'nc-help'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import Model from '../../models/Model'; -import Project from '../../models/Project'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import Model from '../models/Model'; +import Project from '../models/Project'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import { isVirtualCol, ModelTypes, RelationTypes, UITypes } from 'nocodb-sdk'; import { Router } from 'express'; -import Base from '../../models/Base'; -import ModelXcMetaFactory from '../../db/sql-mgr/code/models/xc/ModelXcMetaFactory'; -import Column from '../../models/Column'; -import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; -import { getUniqueColumnAliasName } from '../helpers/getUniqueName'; -import NcHelp from '../../utils/NcHelp'; -import getTableNameAlias, { getColumnNameAlias } from '../helpers/getTableName'; -import mapDefaultDisplayValue from '../helpers/mapDefaultDisplayValue'; -import getColumnUiType from '../helpers/getColumnUiType'; -import { metaApiMetrics } from '../helpers/apiMetrics'; +import Base from '../models/Base'; +import ModelXcMetaFactory from '../db/sql-mgr/code/models/xc/ModelXcMetaFactory'; +import Column from '../models/Column'; +import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; +import { getUniqueColumnAliasName } from '../meta/helpers/getUniqueName'; +import NcHelp from '../utils/NcHelp'; +import getTableNameAlias, { getColumnNameAlias } from '../meta/helpers/getTableName'; +import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; +import getColumnUiType from '../meta/helpers/getColumnUiType'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; export enum MetaDiffType { TABLE_NEW = 'TABLE_NEW', diff --git a/packages/nocodb/src/lib/meta/api/modelVisibilityApis.ts b/packages/nocodb/src/lib/controllers/modelVisibilityApis.ts similarity index 92% rename from packages/nocodb/src/lib/meta/api/modelVisibilityApis.ts rename to packages/nocodb/src/lib/controllers/modelVisibilityApis.ts index 26dc965bb1..0c32f3090b 100644 --- a/packages/nocodb/src/lib/meta/api/modelVisibilityApis.ts +++ b/packages/nocodb/src/lib/controllers/modelVisibilityApis.ts @@ -1,10 +1,10 @@ -import Model from '../../models/Model'; -import ModelRoleVisibility from '../../models/ModelRoleVisibility'; +import Model from '../models/Model'; +import ModelRoleVisibility from '../models/ModelRoleVisibility'; import { Router } from 'express'; import { Tele } from 'nc-help'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; async function xcVisibilityMetaSetAll(req, res) { Tele.emit('evt', { evt_type: 'uiAcl:updated' }); for (const d of req.body) { diff --git a/packages/nocodb/src/lib/meta/api/orgLicenseApis.ts b/packages/nocodb/src/lib/controllers/orgLicenseApis.ts similarity index 75% rename from packages/nocodb/src/lib/meta/api/orgLicenseApis.ts rename to packages/nocodb/src/lib/controllers/orgLicenseApis.ts index ec13f4f9e7..0281145b7a 100644 --- a/packages/nocodb/src/lib/meta/api/orgLicenseApis.ts +++ b/packages/nocodb/src/lib/controllers/orgLicenseApis.ts @@ -1,11 +1,11 @@ import { Router } from 'express'; import { OrgUserRoles } from 'nocodb-sdk'; -import { NC_LICENSE_KEY } from '../../constants'; -import Store from '../../models/Store'; -import Noco from '../../Noco'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { getAjvValidatorMw } from './helpers'; +import { NC_LICENSE_KEY } from '../constants'; +import Store from '../models/Store'; +import Noco from '../Noco'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { getAjvValidatorMw } from '../meta/api/helpers'; async function licenseGet(_req, res) { const license = await Store.get(NC_LICENSE_KEY); diff --git a/packages/nocodb/src/lib/meta/api/orgTokenApis.ts b/packages/nocodb/src/lib/controllers/orgTokenApis.ts similarity index 81% rename from packages/nocodb/src/lib/meta/api/orgTokenApis.ts rename to packages/nocodb/src/lib/controllers/orgTokenApis.ts index 300a804c4e..1e32de65c7 100644 --- a/packages/nocodb/src/lib/meta/api/orgTokenApis.ts +++ b/packages/nocodb/src/lib/controllers/orgTokenApis.ts @@ -1,14 +1,14 @@ import { Request, Response, Router } from 'express'; import { OrgUserRoles } from 'nocodb-sdk'; -import ApiToken from '../../models/ApiToken'; +import ApiToken from '../models/ApiToken'; import { Tele } from 'nc-help'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { NcError } from '../helpers/catchError'; -import getHandler from '../helpers/getHandler'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { PagedResponseImpl } from '../helpers/PagedResponse'; -import { apiTokenListEE } from './ee/orgTokenApis'; -import { getAjvValidatorMw } from './helpers'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { NcError } from '../meta/helpers/catchError'; +import getHandler from '../meta/helpers/getHandler'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import { apiTokenListEE } from '../meta/api/ee/orgTokenApis'; +import { getAjvValidatorMw } from '../meta/api/helpers'; async function apiTokenList(req, res) { const fk_user_id = req.user.id; diff --git a/packages/nocodb/src/lib/meta/api/orgUserApis.ts b/packages/nocodb/src/lib/controllers/orgUserApis.ts similarity index 91% rename from packages/nocodb/src/lib/meta/api/orgUserApis.ts rename to packages/nocodb/src/lib/controllers/orgUserApis.ts index 62c3cdf2aa..0dde26bce6 100644 --- a/packages/nocodb/src/lib/meta/api/orgUserApis.ts +++ b/packages/nocodb/src/lib/controllers/orgUserApis.ts @@ -7,22 +7,22 @@ import { import { v4 as uuidv4 } from 'uuid'; import validator from 'validator'; import { OrgUserRoles } from 'nocodb-sdk'; -import { NC_APP_SETTINGS } from '../../constants'; -import Audit from '../../models/Audit'; -import ProjectUser from '../../models/ProjectUser'; -import Store from '../../models/Store'; -import SyncSource from '../../models/SyncSource'; -import User from '../../models/User'; -import Noco from '../../Noco'; -import { MetaTable } from '../../utils/globals'; +import { NC_APP_SETTINGS } from '../constants'; +import Audit from '../models/Audit'; +import ProjectUser from '../models/ProjectUser'; +import Store from '../models/Store'; +import SyncSource from '../models/SyncSource'; +import User from '../models/User'; +import Noco from '../Noco'; +import { MetaTable } from '../utils/globals'; import { Tele } from 'nc-help'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { NcError } from '../helpers/catchError'; -import { extractProps } from '../helpers/extractProps'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { PagedResponseImpl } from '../helpers/PagedResponse'; -import { randomTokenString } from '../helpers/stringHelpers'; -import { getAjvValidatorMw } from './helpers'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { NcError } from '../meta/helpers/catchError'; +import { extractProps } from '../meta/helpers/extractProps'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import { randomTokenString } from '../meta/helpers/stringHelpers'; +import { getAjvValidatorMw } from '../meta/api/helpers'; import { sendInviteEmail } from './projectUserApis'; async function userList(req, res) { diff --git a/packages/nocodb/src/lib/meta/api/pluginApis.ts b/packages/nocodb/src/lib/controllers/pluginApis.ts similarity index 83% rename from packages/nocodb/src/lib/meta/api/pluginApis.ts rename to packages/nocodb/src/lib/controllers/pluginApis.ts index 6934e17b41..a48ecbf33f 100644 --- a/packages/nocodb/src/lib/meta/api/pluginApis.ts +++ b/packages/nocodb/src/lib/controllers/pluginApis.ts @@ -1,12 +1,12 @@ import { Request, Response, Router } from 'express'; import { Tele } from 'nc-help'; -import { PagedResponseImpl } from '../helpers/PagedResponse'; -import Plugin from '../../models/Plugin'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import Plugin from '../models/Plugin'; import { PluginType } from 'nocodb-sdk'; -import NcPluginMgrv2 from '../helpers/NcPluginMgrv2'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; export async function pluginList(_req: Request, res: Response) { res.json(new PagedResponseImpl(await Plugin.list())); diff --git a/packages/nocodb/src/lib/meta/api/projectApis.ts b/packages/nocodb/src/lib/controllers/projectApis.ts similarity index 90% rename from packages/nocodb/src/lib/meta/api/projectApis.ts rename to packages/nocodb/src/lib/controllers/projectApis.ts index 3c5b536e05..89351269b8 100644 --- a/packages/nocodb/src/lib/meta/api/projectApis.ts +++ b/packages/nocodb/src/lib/controllers/projectApis.ts @@ -1,25 +1,25 @@ import { Request, Response } from 'express'; import { OrgUserRoles, ProjectType } from 'nocodb-sdk'; -import Project from '../../models/Project'; +import Project from '../models/Project'; import { ProjectListType } from 'nocodb-sdk'; import DOMPurify from 'isomorphic-dompurify'; -import { packageVersion } from '../../utils/packageVersion'; +import { packageVersion } from '../utils/packageVersion'; import { Tele } from 'nc-help'; -import { PagedResponseImpl } from '../helpers/PagedResponse'; -import syncMigration from '../helpers/syncMigration'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import ProjectUser from '../../models/ProjectUser'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import syncMigration from '../meta/helpers/syncMigration'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import ProjectUser from '../models/ProjectUser'; import { customAlphabet } from 'nanoid'; -import Noco from '../../Noco'; +import Noco from '../Noco'; import isDocker from 'is-docker'; -import { NcError } from '../helpers/catchError'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { extractPropsAndSanitize } from '../helpers/extractProps'; -import NcConfigFactory from '../../utils/NcConfigFactory'; +import { NcError } from '../meta/helpers/catchError'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { extractPropsAndSanitize } from '../meta/helpers/extractProps'; +import NcConfigFactory from '../utils/NcConfigFactory'; import { promisify } from 'util'; -import { getAjvValidatorMw, populateMeta } from './helpers'; -import Filter from '../../models/Filter'; +import { getAjvValidatorMw, populateMeta } from '../meta/api/helpers'; +import Filter from '../models/Filter'; const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 4); diff --git a/packages/nocodb/src/lib/meta/api/projectUserApis.ts b/packages/nocodb/src/lib/controllers/projectUserApis.ts similarity index 92% rename from packages/nocodb/src/lib/meta/api/projectUserApis.ts rename to packages/nocodb/src/lib/controllers/projectUserApis.ts index 3f0610fee6..270db703ed 100644 --- a/packages/nocodb/src/lib/meta/api/projectUserApis.ts +++ b/packages/nocodb/src/lib/controllers/projectUserApis.ts @@ -1,23 +1,23 @@ import { OrgUserRoles } from 'nocodb-sdk'; import { Tele } from 'nc-help'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { Router } from 'express'; -import { PagedResponseImpl } from '../helpers/PagedResponse'; -import ProjectUser from '../../models/ProjectUser'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import ProjectUser from '../models/ProjectUser'; import validator from 'validator'; -import { NcError } from '../helpers/catchError'; +import { NcError } from '../meta/helpers/catchError'; import { v4 as uuidv4 } from 'uuid'; -import User from '../../models/User'; -import Audit from '../../models/Audit'; -import NocoCache from '../../cache/NocoCache'; -import { CacheGetType, CacheScope, MetaTable } from '../../utils/globals'; +import User from '../models/User'; +import Audit from '../models/Audit'; +import NocoCache from '../cache/NocoCache'; +import { CacheGetType, CacheScope, MetaTable } from '../utils/globals'; import * as ejs from 'ejs'; -import NcPluginMgrv2 from '../helpers/NcPluginMgrv2'; -import Noco from '../../Noco'; +import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; +import Noco from '../Noco'; import { PluginCategory } from 'nocodb-sdk'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { randomTokenString } from '../helpers/stringHelpers'; -import { getAjvValidatorMw } from './helpers'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { randomTokenString } from '../meta/helpers/stringHelpers'; +import { getAjvValidatorMw } from '../meta/api/helpers'; async function userList(req, res) { res.json({ diff --git a/packages/nocodb/src/lib/meta/api/publicApis/index.ts b/packages/nocodb/src/lib/controllers/publicApis/index.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/publicApis/index.ts rename to packages/nocodb/src/lib/controllers/publicApis/index.ts diff --git a/packages/nocodb/src/lib/meta/api/publicApis/publicDataApis.ts b/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts similarity index 94% rename from packages/nocodb/src/lib/meta/api/publicApis/publicDataApis.ts rename to packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts index bb678b10f3..cc81ef56a3 100644 --- a/packages/nocodb/src/lib/meta/api/publicApis/publicDataApis.ts +++ b/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts @@ -1,24 +1,24 @@ import { Request, Response, Router } from 'express'; -import Model from '../../../models/Model'; +import Model from '../../models/Model'; import { nocoExecute } from 'nc-help'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; -import { PagedResponseImpl } from '../../helpers/PagedResponse'; -import View from '../../../models/View'; -import catchError, { NcError } from '../../helpers/catchError'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import View from '../../models/View'; +import catchError, { NcError } from '../../meta/helpers/catchError'; import multer from 'multer'; import { ErrorMessages, UITypes, ViewTypes } from 'nocodb-sdk'; -import Column from '../../../models/Column'; -import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; -import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2'; +import Column from '../../models/Column'; +import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; +import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; import path from 'path'; import { nanoid } from 'nanoid'; -import { mimeIcons } from '../../../utils/mimeTypes'; +import { mimeIcons } from '../../utils/mimeTypes'; import slash from 'slash'; import { sanitizeUrlPath } from '../attachmentApis'; -import getAst from '../../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; import { getColumnByIdOrName } from '../dataApis/helpers'; -import { NC_ATTACHMENT_FIELD_SIZE } from '../../../constants'; +import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants'; export async function dataList(req: Request, res: Response) { try { diff --git a/packages/nocodb/src/lib/meta/api/publicApis/publicDataExportApis.ts b/packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts similarity index 92% rename from packages/nocodb/src/lib/meta/api/publicApis/publicDataExportApis.ts rename to packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts index 936cc68c9b..42ae1d91a6 100644 --- a/packages/nocodb/src/lib/meta/api/publicApis/publicDataExportApis.ts +++ b/packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts @@ -1,17 +1,17 @@ import { Request, Response, Router } from 'express'; import * as XLSX from 'xlsx'; -import View from '../../../models/View'; -import Model from '../../../models/Model'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; +import View from '../../models/View'; +import Model from '../../models/Model'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; import { nocoExecute } from 'nc-help'; import papaparse from 'papaparse'; import { ErrorMessages, isSystemColumn, UITypes, ViewTypes } from 'nocodb-sdk'; -import Column from '../../../models/Column'; -import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; -import LookupColumn from '../../../models/LookupColumn'; -import catchError, { NcError } from '../../helpers/catchError'; -import getAst from '../../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import Column from '../../models/Column'; +import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; +import LookupColumn from '../../models/LookupColumn'; +import catchError, { NcError } from '../../meta/helpers/catchError'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; async function exportExcel(req: Request, res: Response) { const view = await View.getByUUID(req.params.publicDataUuid); diff --git a/packages/nocodb/src/lib/meta/api/publicApis/publicMetaApis.ts b/packages/nocodb/src/lib/controllers/publicApis/publicMetaApis.ts similarity index 88% rename from packages/nocodb/src/lib/meta/api/publicApis/publicMetaApis.ts rename to packages/nocodb/src/lib/controllers/publicApis/publicMetaApis.ts index 9bb4aaf946..7d5af6036f 100644 --- a/packages/nocodb/src/lib/meta/api/publicApis/publicMetaApis.ts +++ b/packages/nocodb/src/lib/controllers/publicApis/publicMetaApis.ts @@ -1,17 +1,17 @@ import { Request, Response, Router } from 'express'; -import catchError, { NcError } from '../../helpers/catchError'; -import View from '../../../models/View'; -import Model from '../../../models/Model'; +import catchError, { NcError } from '../../meta/helpers/catchError'; +import View from '../../models/View'; +import Model from '../../models/Model'; import { ErrorMessages, LinkToAnotherRecordType, RelationTypes, UITypes, } from 'nocodb-sdk'; -import Column from '../../../models/Column'; -import Base from '../../../models/Base'; -import Project from '../../../models/Project'; -import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; +import Column from '../../models/Column'; +import Base from '../../models/Base'; +import Project from '../../models/Project'; +import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; export async function viewMetaGet(req: Request, res: Response) { const view: View & { diff --git a/packages/nocodb/src/lib/meta/api/sharedBaseApis.ts b/packages/nocodb/src/lib/controllers/sharedBaseApis.ts similarity index 93% rename from packages/nocodb/src/lib/meta/api/sharedBaseApis.ts rename to packages/nocodb/src/lib/controllers/sharedBaseApis.ts index c232327aa8..a9bfcfbb48 100644 --- a/packages/nocodb/src/lib/meta/api/sharedBaseApis.ts +++ b/packages/nocodb/src/lib/controllers/sharedBaseApis.ts @@ -1,10 +1,10 @@ import { Router } from 'express'; import { Tele } from 'nc-help'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { v4 as uuidv4 } from 'uuid'; -import Project from '../../models/Project'; -import { NcError } from '../helpers/catchError'; -import { getAjvValidatorMw } from './helpers'; +import Project from '../models/Project'; +import { NcError } from '../meta/helpers/catchError'; +import { getAjvValidatorMw } from '../meta/api/helpers'; // todo: load from config const config = { dashboardPath: '/nc', diff --git a/packages/nocodb/src/lib/meta/api/sortApis.ts b/packages/nocodb/src/lib/controllers/sortApis.ts similarity index 82% rename from packages/nocodb/src/lib/meta/api/sortApis.ts rename to packages/nocodb/src/lib/controllers/sortApis.ts index 5341d07cba..c1ee743b51 100644 --- a/packages/nocodb/src/lib/meta/api/sortApis.ts +++ b/packages/nocodb/src/lib/controllers/sortApis.ts @@ -1,18 +1,18 @@ import { Request, Response, Router } from 'express'; // @ts-ignore -import Model from '../../models/Model'; +import Model from '../models/Model'; import { Tele } from 'nc-help'; // @ts-ignore -import { PagedResponseImpl } from '../helpers/PagedResponse'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import { SortListType, SortType, TableType } from 'nocodb-sdk'; // @ts-ignore -import ProjectMgrv2 from '../../db/sql-mgr/v2/ProjectMgrv2'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; // @ts-ignore -import Project from '../../models/Project'; -import Sort from '../../models/Sort'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import Project from '../models/Project'; +import Sort from '../models/Sort'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { getAjvValidatorMw } from '../meta/api/helpers'; // @ts-ignore export async function sortGet(req: Request, res: Response) {} diff --git a/packages/nocodb/src/lib/meta/api/swagger/helpers/getPaths.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts similarity index 88% rename from packages/nocodb/src/lib/meta/api/swagger/helpers/getPaths.ts rename to packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts index a1d3d0b8f8..7576e52a2d 100644 --- a/packages/nocodb/src/lib/meta/api/swagger/helpers/getPaths.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts @@ -1,6 +1,6 @@ -import Noco from '../../../../Noco'; -import Model from '../../../../models/Model'; -import Project from '../../../../models/Project'; +import Noco from '../../../Noco'; +import Model from '../../../models/Model'; +import Project from '../../../models/Project'; import { getModelPaths, getViewPaths } from './templates/paths'; import { SwaggerColumn } from './getSwaggerColumnMetas'; import { SwaggerView } from './getSwaggerJSON'; diff --git a/packages/nocodb/src/lib/meta/api/swagger/helpers/getSchemas.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts similarity index 88% rename from packages/nocodb/src/lib/meta/api/swagger/helpers/getSchemas.ts rename to packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts index a62d19b220..78dc05abbb 100644 --- a/packages/nocodb/src/lib/meta/api/swagger/helpers/getSchemas.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts @@ -1,6 +1,6 @@ -import Noco from '../../../../Noco'; -import Model from '../../../../models/Model'; -import Project from '../../../../models/Project'; +import Noco from '../../../Noco'; +import Model from '../../../models/Model'; +import Project from '../../../models/Project'; import { getModelSchemas, getViewSchemas } from './templates/schemas'; import { SwaggerColumn } from './getSwaggerColumnMetas'; import { SwaggerView } from './getSwaggerJSON'; diff --git a/packages/nocodb/src/lib/meta/api/swagger/helpers/getSwaggerColumnMetas.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts similarity index 83% rename from packages/nocodb/src/lib/meta/api/swagger/helpers/getSwaggerColumnMetas.ts rename to packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts index 6ea1b312d6..46d2f202ac 100644 --- a/packages/nocodb/src/lib/meta/api/swagger/helpers/getSwaggerColumnMetas.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts @@ -1,9 +1,9 @@ import { UITypes } from 'nocodb-sdk'; -import LinkToAnotherRecordColumn from '../../../../models/LinkToAnotherRecordColumn'; -import SwaggerTypes from '../../../../db/sql-mgr/code/routers/xc-ts/SwaggerTypes'; -import Column from '../../../../models/Column'; -import Noco from '../../../../Noco'; -import Project from '../../../../models/Project'; +import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; +import SwaggerTypes from '../../../db/sql-mgr/code/routers/xc-ts/SwaggerTypes'; +import Column from '../../../models/Column'; +import Noco from '../../../Noco'; +import Project from '../../../models/Project'; export default async ( columns: Column[], diff --git a/packages/nocodb/src/lib/meta/api/swagger/helpers/getSwaggerJSON.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts similarity index 80% rename from packages/nocodb/src/lib/meta/api/swagger/helpers/getSwaggerJSON.ts rename to packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts index 19c09a5514..4a17db4504 100644 --- a/packages/nocodb/src/lib/meta/api/swagger/helpers/getSwaggerJSON.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts @@ -1,15 +1,15 @@ -import FormViewColumn from '../../../../models/FormViewColumn'; -import GalleryViewColumn from '../../../../models/GalleryViewColumn'; -import Noco from '../../../../Noco'; -import Model from '../../../../models/Model'; +import FormViewColumn from '../../../models/FormViewColumn'; +import GalleryViewColumn from '../../../models/GalleryViewColumn'; +import Noco from '../../../Noco'; +import Model from '../../../models/Model'; import swaggerBase from './swagger-base.json'; import getPaths from './getPaths'; import getSchemas from './getSchemas'; -import Project from '../../../../models/Project'; +import Project from '../../../models/Project'; import getSwaggerColumnMetas from './getSwaggerColumnMetas'; import { ViewTypes } from 'nocodb-sdk'; -import GridViewColumn from '../../../../models/GridViewColumn'; -import View from '../../../../models/View'; +import GridViewColumn from '../../../models/GridViewColumn'; +import View from '../../../models/View'; export default async function getSwaggerJSON( project: Project, diff --git a/packages/nocodb/src/lib/meta/api/swagger/helpers/swagger-base.json b/packages/nocodb/src/lib/controllers/swagger/helpers/swagger-base.json similarity index 100% rename from packages/nocodb/src/lib/meta/api/swagger/helpers/swagger-base.json rename to packages/nocodb/src/lib/controllers/swagger/helpers/swagger-base.json diff --git a/packages/nocodb/src/lib/meta/api/swagger/helpers/templates/headers.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/swagger/helpers/templates/headers.ts rename to packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts diff --git a/packages/nocodb/src/lib/meta/api/swagger/helpers/templates/params.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts similarity index 98% rename from packages/nocodb/src/lib/meta/api/swagger/helpers/templates/params.ts rename to packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts index 891ecfd10f..e9c43901a5 100644 --- a/packages/nocodb/src/lib/meta/api/swagger/helpers/templates/params.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts @@ -1,6 +1,6 @@ import { SwaggerColumn } from '../getSwaggerColumnMetas'; import { RelationTypes, UITypes } from 'nocodb-sdk'; -import LinkToAnotherRecordColumn from '../../../../../models/LinkToAnotherRecordColumn'; +import LinkToAnotherRecordColumn from '../../../../models/LinkToAnotherRecordColumn'; export const rowIdParam = { schema: { diff --git a/packages/nocodb/src/lib/meta/api/swagger/helpers/templates/paths.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/paths.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/swagger/helpers/templates/paths.ts rename to packages/nocodb/src/lib/controllers/swagger/helpers/templates/paths.ts diff --git a/packages/nocodb/src/lib/meta/api/swagger/helpers/templates/schemas.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/schemas.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/swagger/helpers/templates/schemas.ts rename to packages/nocodb/src/lib/controllers/swagger/helpers/templates/schemas.ts diff --git a/packages/nocodb/src/lib/meta/api/swagger/redocHtml.ts b/packages/nocodb/src/lib/controllers/swagger/redocHtml.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/swagger/redocHtml.ts rename to packages/nocodb/src/lib/controllers/swagger/redocHtml.ts diff --git a/packages/nocodb/src/lib/meta/api/swagger/swaggerApis.ts b/packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts similarity index 86% rename from packages/nocodb/src/lib/meta/api/swagger/swaggerApis.ts rename to packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts index c8cb637ba1..ef44542646 100644 --- a/packages/nocodb/src/lib/meta/api/swagger/swaggerApis.ts +++ b/packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts @@ -1,10 +1,10 @@ // @ts-ignore -import catchError, { NcError } from '../../helpers/catchError'; +import catchError, { NcError } from '../../meta/helpers/catchError'; import { Router } from 'express'; -import Model from '../../../models/Model'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import Model from '../../models/Model'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; import getSwaggerJSON from './helpers/getSwaggerJSON'; -import Project from '../../../models/Project'; +import Project from '../../models/Project'; import getSwaggerHtml from './swaggerHtml'; import getRedocHtml from './redocHtml'; diff --git a/packages/nocodb/src/lib/meta/api/swagger/swaggerHtml.ts b/packages/nocodb/src/lib/controllers/swagger/swaggerHtml.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/swagger/swaggerHtml.ts rename to packages/nocodb/src/lib/controllers/swagger/swaggerHtml.ts diff --git a/packages/nocodb/src/lib/meta/api/sync/helpers/EntityMap.ts b/packages/nocodb/src/lib/controllers/sync/helpers/EntityMap.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/sync/helpers/EntityMap.ts rename to packages/nocodb/src/lib/controllers/sync/helpers/EntityMap.ts diff --git a/packages/nocodb/src/lib/meta/api/sync/helpers/NocoSyncDestAdapter.ts b/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncDestAdapter.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/sync/helpers/NocoSyncDestAdapter.ts rename to packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncDestAdapter.ts diff --git a/packages/nocodb/src/lib/meta/api/sync/helpers/NocoSyncSourceAdapter.ts b/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncSourceAdapter.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/sync/helpers/NocoSyncSourceAdapter.ts rename to packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncSourceAdapter.ts diff --git a/packages/nocodb/src/lib/meta/api/sync/helpers/fetchAT.ts b/packages/nocodb/src/lib/controllers/sync/helpers/fetchAT.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/sync/helpers/fetchAT.ts rename to packages/nocodb/src/lib/controllers/sync/helpers/fetchAT.ts diff --git a/packages/nocodb/src/lib/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/controllers/sync/helpers/job.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/sync/helpers/job.ts rename to packages/nocodb/src/lib/controllers/sync/helpers/job.ts diff --git a/packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/controllers/sync/helpers/readAndProcessData.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts rename to packages/nocodb/src/lib/controllers/sync/helpers/readAndProcessData.ts diff --git a/packages/nocodb/src/lib/meta/api/sync/helpers/syncMap.ts b/packages/nocodb/src/lib/controllers/sync/helpers/syncMap.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/sync/helpers/syncMap.ts rename to packages/nocodb/src/lib/controllers/sync/helpers/syncMap.ts diff --git a/packages/nocodb/src/lib/meta/api/sync/importApis.ts b/packages/nocodb/src/lib/controllers/sync/importApis.ts similarity index 94% rename from packages/nocodb/src/lib/meta/api/sync/importApis.ts rename to packages/nocodb/src/lib/controllers/sync/importApis.ts index 92ab241117..029caa0657 100644 --- a/packages/nocodb/src/lib/meta/api/sync/importApis.ts +++ b/packages/nocodb/src/lib/controllers/sync/importApis.ts @@ -1,12 +1,12 @@ import { Request, Router } from 'express'; // import { Queue } from 'bullmq'; // import axios from 'axios'; -import catchError, { NcError } from '../../helpers/catchError'; +import catchError, { NcError } from '../../meta/helpers/catchError'; import { Server } from 'socket.io'; -import NocoJobs from '../../../jobs/NocoJobs'; +import NocoJobs from '../../jobs/NocoJobs'; import job, { AirtableSyncConfig } from './helpers/job'; -import SyncSource from '../../../models/SyncSource'; -import Noco from '../../../Noco'; +import SyncSource from '../../models/SyncSource'; +import Noco from '../../Noco'; import { genJwt } from '../userApi/helpers'; const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB'; const AIRTABLE_PROGRESS_JOB = 'AIRTABLE_PROGRESS_JOB'; diff --git a/packages/nocodb/src/lib/meta/api/sync/syncSourceApis.ts b/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts similarity index 89% rename from packages/nocodb/src/lib/meta/api/sync/syncSourceApis.ts rename to packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts index f9e761ff71..2720f8fd88 100644 --- a/packages/nocodb/src/lib/meta/api/sync/syncSourceApis.ts +++ b/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts @@ -1,10 +1,10 @@ import { Request, Response, Router } from 'express'; -import SyncSource from '../../../models/SyncSource'; +import SyncSource from '../../models/SyncSource'; import { Tele } from 'nc-help'; -import { PagedResponseImpl } from '../../helpers/PagedResponse'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import Project from '../../../models/Project'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import Project from '../../models/Project'; export async function syncSourceList(req: Request, res: Response) { // todo: pagination diff --git a/packages/nocodb/src/lib/meta/api/tableApis.ts b/packages/nocodb/src/lib/controllers/tableApis.ts similarity index 92% rename from packages/nocodb/src/lib/meta/api/tableApis.ts rename to packages/nocodb/src/lib/controllers/tableApis.ts index 4c5d6856a5..db15fac179 100644 --- a/packages/nocodb/src/lib/meta/api/tableApis.ts +++ b/packages/nocodb/src/lib/controllers/tableApis.ts @@ -1,7 +1,7 @@ import { Request, Response, Router } from 'express'; -import Model from '../../models/Model'; +import Model from '../models/Model'; import { Tele } from 'nc-help'; -import { PagedResponseImpl } from '../helpers/PagedResponse'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import DOMPurify from 'isomorphic-dompurify'; import { AuditOperationSubTypes, @@ -14,22 +14,22 @@ import { TableType, UITypes, } from 'nocodb-sdk'; -import ProjectMgrv2 from '../../db/sql-mgr/v2/ProjectMgrv2'; -import Project from '../../models/Project'; -import Audit from '../../models/Audit'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { getAjvValidatorMw } from './helpers'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; +import Project from '../models/Project'; +import Audit from '../models/Audit'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { getAjvValidatorMw } from '../meta/api/helpers'; import { xcVisibilityMetaGet } from './modelVisibilityApis'; -import View from '../../models/View'; -import getColumnPropsFromUIDT from '../helpers/getColumnPropsFromUIDT'; -import mapDefaultDisplayValue from '../helpers/mapDefaultDisplayValue'; -import { NcError } from '../helpers/catchError'; -import getTableNameAlias, { getColumnNameAlias } from '../helpers/getTableName'; -import Column from '../../models/Column'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; -import getColumnUiType from '../helpers/getColumnUiType'; -import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; -import { metaApiMetrics } from '../helpers/apiMetrics'; +import View from '../models/View'; +import getColumnPropsFromUIDT from '../meta/helpers/getColumnPropsFromUIDT'; +import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; +import { NcError } from '../meta/helpers/catchError'; +import getTableNameAlias, { getColumnNameAlias } from '../meta/helpers/getTableName'; +import Column from '../models/Column'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import getColumnUiType from '../meta/helpers/getColumnUiType'; +import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; export async function tableGet(req: Request, res: Response) { const table = await Model.getWithInfo({ diff --git a/packages/nocodb/src/lib/meta/api/testApis.ts b/packages/nocodb/src/lib/controllers/testApis.ts similarity index 85% rename from packages/nocodb/src/lib/meta/api/testApis.ts rename to packages/nocodb/src/lib/controllers/testApis.ts index 506f9d4474..08ea1e780e 100644 --- a/packages/nocodb/src/lib/meta/api/testApis.ts +++ b/packages/nocodb/src/lib/controllers/testApis.ts @@ -1,5 +1,5 @@ import { Request, Router } from 'express'; -import { TestResetService } from '../../services/test/TestResetService'; +import { TestResetService } from '../services/test/TestResetService'; export async function reset(req: Request, res) { const service = new TestResetService({ diff --git a/packages/nocodb/src/lib/meta/api/userApi/helpers.ts b/packages/nocodb/src/lib/controllers/userApi/helpers.ts similarity index 83% rename from packages/nocodb/src/lib/meta/api/userApi/helpers.ts rename to packages/nocodb/src/lib/controllers/userApi/helpers.ts index e7686b2a81..387b51170e 100644 --- a/packages/nocodb/src/lib/meta/api/userApi/helpers.ts +++ b/packages/nocodb/src/lib/controllers/userApi/helpers.ts @@ -1,7 +1,7 @@ import * as jwt from 'jsonwebtoken'; import crypto from 'crypto'; -import User from '../../../models/User'; -import { NcConfig } from '../../../../interface/config'; +import User from '../../models/User'; +import { NcConfig } from '../../../interface/config'; export function genJwt(user: User, config: NcConfig) { return jwt.sign( diff --git a/packages/nocodb/src/lib/meta/api/userApi/index.ts b/packages/nocodb/src/lib/controllers/userApi/index.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/userApi/index.ts rename to packages/nocodb/src/lib/controllers/userApi/index.ts diff --git a/packages/nocodb/src/lib/meta/api/userApi/initAdminFromEnv.ts b/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts similarity index 97% rename from packages/nocodb/src/lib/meta/api/userApi/initAdminFromEnv.ts rename to packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts index 8f9bfdce74..c39328bfb2 100644 --- a/packages/nocodb/src/lib/meta/api/userApi/initAdminFromEnv.ts +++ b/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts @@ -1,14 +1,14 @@ -import User from '../../../models/User'; +import User from '../../models/User'; import { v4 as uuidv4 } from 'uuid'; import { promisify } from 'util'; import bcrypt from 'bcryptjs'; -import Noco from '../../../Noco'; -import { CacheScope, MetaTable } from '../../../utils/globals'; -import ProjectUser from '../../../models/ProjectUser'; +import Noco from '../../Noco'; +import { CacheScope, MetaTable } from '../../utils/globals'; +import ProjectUser from '../../models/ProjectUser'; import { validatePassword } from 'nocodb-sdk'; import boxen from 'boxen'; -import NocoCache from '../../../cache/NocoCache'; +import NocoCache from '../../cache/NocoCache'; import { Tele } from 'nc-help'; const { isEmail } = require('validator'); diff --git a/packages/nocodb/src/lib/meta/api/userApi/initStrategies.ts b/packages/nocodb/src/lib/controllers/userApi/initStrategies.ts similarity index 96% rename from packages/nocodb/src/lib/meta/api/userApi/initStrategies.ts rename to packages/nocodb/src/lib/controllers/userApi/initStrategies.ts index b40f2c58d1..85fc8bbf9c 100644 --- a/packages/nocodb/src/lib/meta/api/userApi/initStrategies.ts +++ b/packages/nocodb/src/lib/controllers/userApi/initStrategies.ts @@ -1,6 +1,6 @@ import { OrgUserRoles } from 'nocodb-sdk'; -import User from '../../../models/User'; -import ProjectUser from '../../../models/ProjectUser'; +import User from '../../models/User'; +import ProjectUser from '../../models/ProjectUser'; import { promisify } from 'util'; import { Strategy as CustomStrategy } from 'passport-custom'; import passport from 'passport'; @@ -17,12 +17,12 @@ const jwtOptions = { }; import bcrypt from 'bcryptjs'; -import Project from '../../../models/Project'; -import NocoCache from '../../../cache/NocoCache'; -import { CacheGetType, CacheScope } from '../../../utils/globals'; -import ApiToken from '../../../models/ApiToken'; -import Noco from '../../../Noco'; -import Plugin from '../../../models/Plugin'; +import Project from '../../models/Project'; +import NocoCache from '../../cache/NocoCache'; +import { CacheGetType, CacheScope } from '../../utils/globals'; +import ApiToken from '../../models/ApiToken'; +import Noco from '../../Noco'; +import Plugin from '../../models/Plugin'; import { registerNewUserIfAllowed } from './userApis'; export function initStrategies(router): void { diff --git a/packages/nocodb/src/lib/meta/api/userApi/ui/auth/emailVerify.ts b/packages/nocodb/src/lib/controllers/userApi/ui/auth/emailVerify.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/userApi/ui/auth/emailVerify.ts rename to packages/nocodb/src/lib/controllers/userApi/ui/auth/emailVerify.ts diff --git a/packages/nocodb/src/lib/meta/api/userApi/ui/auth/resetPassword.ts b/packages/nocodb/src/lib/controllers/userApi/ui/auth/resetPassword.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/userApi/ui/auth/resetPassword.ts rename to packages/nocodb/src/lib/controllers/userApi/ui/auth/resetPassword.ts diff --git a/packages/nocodb/src/lib/meta/api/userApi/ui/emailTemplates/forgotPassword.ts b/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/forgotPassword.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/userApi/ui/emailTemplates/forgotPassword.ts rename to packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/forgotPassword.ts diff --git a/packages/nocodb/src/lib/meta/api/userApi/ui/emailTemplates/invite.ts b/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/invite.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/userApi/ui/emailTemplates/invite.ts rename to packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/invite.ts diff --git a/packages/nocodb/src/lib/meta/api/userApi/ui/emailTemplates/verify.ts b/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/verify.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/userApi/ui/emailTemplates/verify.ts rename to packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/verify.ts diff --git a/packages/nocodb/src/lib/meta/api/userApi/userApis.ts b/packages/nocodb/src/lib/controllers/userApi/userApis.ts similarity index 96% rename from packages/nocodb/src/lib/meta/api/userApi/userApis.ts rename to packages/nocodb/src/lib/controllers/userApi/userApis.ts index 809b96a6c1..b41ca7dfa1 100644 --- a/packages/nocodb/src/lib/meta/api/userApi/userApis.ts +++ b/packages/nocodb/src/lib/controllers/userApi/userApis.ts @@ -1,30 +1,30 @@ import { Request, Response } from 'express'; import { TableType, validatePassword } from 'nocodb-sdk'; import { OrgUserRoles } from 'nocodb-sdk'; -import { NC_APP_SETTINGS } from '../../../constants'; -import Store from '../../../models/Store'; +import { NC_APP_SETTINGS } from '../../constants'; +import Store from '../../models/Store'; import { Tele } from 'nc-help'; -import catchError, { NcError } from '../../helpers/catchError'; +import catchError, { NcError } from '../../meta/helpers/catchError'; const { isEmail } = require('validator'); import * as ejs from 'ejs'; import bcrypt from 'bcryptjs'; import { promisify } from 'util'; -import User from '../../../models/User'; +import User from '../../models/User'; const { v4: uuidv4 } = require('uuid'); -import Audit from '../../../models/Audit'; -import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2'; +import Audit from '../../models/Audit'; +import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; import passport from 'passport'; -import extractProjectIdAndAuthenticate from '../../helpers/extractProjectIdAndAuthenticate'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import { MetaTable } from '../../../utils/globals'; -import Noco from '../../../Noco'; -import { getAjvValidatorMw } from '../helpers'; +import extractProjectIdAndAuthenticate from '../../meta/helpers/extractProjectIdAndAuthenticate'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import { MetaTable } from '../../utils/globals'; +import Noco from '../../Noco'; +import { getAjvValidatorMw } from '../../meta/api/helpers'; import { genJwt } from './helpers'; -import { randomTokenString } from '../../helpers/stringHelpers'; +import { randomTokenString } from '../../meta/helpers/stringHelpers'; export async function registerNewUserIfAllowed({ firstname, diff --git a/packages/nocodb/src/lib/meta/api/utilApis.ts b/packages/nocodb/src/lib/controllers/utilApis.ts similarity index 95% rename from packages/nocodb/src/lib/meta/api/utilApis.ts rename to packages/nocodb/src/lib/controllers/utilApis.ts index 8bafe86ce4..93ad927e1c 100644 --- a/packages/nocodb/src/lib/meta/api/utilApis.ts +++ b/packages/nocodb/src/lib/controllers/utilApis.ts @@ -3,20 +3,20 @@ import { Request, Response } from 'express'; import { compareVersions, validate } from 'compare-versions'; import { ViewTypes } from 'nocodb-sdk'; -import Project from '../../models/Project'; -import Noco from '../../Noco'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; -import { MetaTable } from '../../utils/globals'; -import { packageVersion } from '../../utils/packageVersion'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import SqlMgrv2 from '../../db/sql-mgr/v2/SqlMgrv2'; +import Project from '../models/Project'; +import Noco from '../Noco'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import { MetaTable } from '../utils/globals'; +import { packageVersion } from '../utils/packageVersion'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2'; import NcConfigFactory, { defaultConnectionConfig, -} from '../../utils/NcConfigFactory'; -import User from '../../models/User'; -import catchError from '../helpers/catchError'; +} from '../utils/NcConfigFactory'; +import User from '../models/User'; +import catchError from '../meta/helpers/catchError'; import axios from 'axios'; -import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants'; +import { NC_ATTACHMENT_FIELD_SIZE } from '../constants'; const versionCache = { releaseVersion: null, diff --git a/packages/nocodb/src/lib/meta/api/viewApis.ts b/packages/nocodb/src/lib/controllers/viewApis.ts similarity index 91% rename from packages/nocodb/src/lib/meta/api/viewApis.ts rename to packages/nocodb/src/lib/controllers/viewApis.ts index b062a2b2ef..da8e387878 100644 --- a/packages/nocodb/src/lib/meta/api/viewApis.ts +++ b/packages/nocodb/src/lib/controllers/viewApis.ts @@ -1,19 +1,19 @@ import { Request, Response, Router } from 'express'; // @ts-ignore -import Model from '../../models/Model'; +import Model from '../models/Model'; import { Tele } from 'nc-help'; // @ts-ignore -import { PagedResponseImpl } from '../helpers/PagedResponse'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; // @ts-ignore import { Table, TableReq, ViewList } from 'nocodb-sdk'; // @ts-ignore -import ProjectMgrv2 from '../../db/sql-mgr/v2/ProjectMgrv2'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; // @ts-ignore -import Project from '../../models/Project'; -import View from '../../models/View'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; +import Project from '../models/Project'; +import View from '../models/View'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { xcVisibilityMetaGet } from './modelVisibilityApis'; -import { metaApiMetrics } from '../helpers/apiMetrics'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; // @ts-ignore export async function viewGet(req: Request, res: Response) {} diff --git a/packages/nocodb/src/lib/meta/api/viewColumnApis.ts b/packages/nocodb/src/lib/controllers/viewColumnApis.ts similarity index 88% rename from packages/nocodb/src/lib/meta/api/viewColumnApis.ts rename to packages/nocodb/src/lib/controllers/viewColumnApis.ts index 2fc4df7625..a7c7fda7eb 100644 --- a/packages/nocodb/src/lib/meta/api/viewColumnApis.ts +++ b/packages/nocodb/src/lib/controllers/viewColumnApis.ts @@ -1,8 +1,8 @@ import { Request, Response, Router } from 'express'; -import View from '../../models/View'; +import View from '../models/View'; import { Tele } from 'nc-help'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; export async function columnList(req: Request, res: Response) { res.json(await View.getColumns(req.params.viewId)); diff --git a/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts b/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts index ffe246a131..7f4e7bc41f 100644 --- a/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts +++ b/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts @@ -11,7 +11,7 @@ import getTableNameAlias, { import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; import getColumnUiType from '../../helpers/getColumnUiType'; import mapDefaultDisplayValue from '../../helpers/mapDefaultDisplayValue'; -import { extractAndGenerateManyToManyRelations } from '../metaDiffApis'; +import { extractAndGenerateManyToManyRelations } from '../../../controllers/metaDiffApis'; import { ModelTypes, UITypes, ViewTypes } from 'nocodb-sdk'; import { IGNORE_TABLES } from '../../../utils/common/BaseApiBuilder'; diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index 67e6203407..c7285c7330 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -1,39 +1,39 @@ import { Tele } from 'nc-help'; -import orgLicenseApis from './orgLicenseApis'; -import orgTokenApis from './orgTokenApis'; -import orgUserApis from './orgUserApis'; -import projectApis from './projectApis'; -import baseApis from './baseApis'; -import tableApis from './tableApis'; -import columnApis from './columnApis'; +import orgLicenseApis from '../../controllers/orgLicenseApis'; +import orgTokenApis from '../../controllers/orgTokenApis'; +import orgUserApis from '../../controllers/orgUserApis'; +import projectApis from '../../controllers/projectApis'; +import baseApis from '../../controllers/baseApis'; +import tableApis from '../../controllers/tableApis'; +import columnApis from '../../controllers/columnApis'; import { Router } from 'express'; -import sortApis from './sortApis'; -import filterApis from './filterApis'; -import viewColumnApis from './viewColumnApis'; -import gridViewApis from './gridViewApis'; -import viewApis from './viewApis'; -import galleryViewApis from './galleryViewApis'; -import formViewApis from './formViewApis'; -import formViewColumnApis from './formViewColumnApis'; -import attachmentApis from './attachmentApis'; -import exportApis from './exportApis'; -import auditApis from './auditApis'; -import hookApis from './hookApis'; -import pluginApis from './pluginApis'; -import gridViewColumnApis from './gridViewColumnApis'; -import kanbanViewApis from './kanbanViewApis'; -import { userApis } from './userApi'; +import sortApis from '../../controllers/sortApis'; +import filterApis from '../../controllers/filterApis'; +import viewColumnApis from '../../controllers/viewColumnApis'; +import gridViewApis from '../../controllers/gridViewApis'; +import viewApis from '../../controllers/viewApis'; +import galleryViewApis from '../../controllers/galleryViewApis'; +import formViewApis from '../../controllers/formViewApis'; +import formViewColumnApis from '../../controllers/formViewColumnApis'; +import attachmentApis from '../../controllers/attachmentApis'; +import exportApis from '../../controllers/exportApis'; +import auditApis from '../../controllers/auditApis'; +import hookApis from '../../controllers/hookApis'; +import pluginApis from '../../controllers/pluginApis'; +import gridViewColumnApis from '../../controllers/gridViewColumnApis'; +import kanbanViewApis from '../../controllers/kanbanViewApis'; +import { userApis } from '../../controllers/userApi'; // import extractProjectIdAndAuthenticate from './helpers/extractProjectIdAndAuthenticate'; -import utilApis from './utilApis'; -import projectUserApis from './projectUserApis'; -import sharedBaseApis from './sharedBaseApis'; -import { initStrategies } from './userApi/initStrategies'; -import modelVisibilityApis from './modelVisibilityApis'; -import metaDiffApis from './metaDiffApis'; -import cacheApis from './cacheApis'; -import apiTokenApis from './apiTokenApis'; -import hookFilterApis from './hookFilterApis'; -import testApis from './testApis'; +import utilApis from '../../controllers/utilApis'; +import projectUserApis from '../../controllers/projectUserApis'; +import sharedBaseApis from '../../controllers/sharedBaseApis'; +import { initStrategies } from '../../controllers/userApi/initStrategies'; +import modelVisibilityApis from '../../controllers/modelVisibilityApis'; +import metaDiffApis from '../../controllers/metaDiffApis'; +import cacheApis from '../../controllers/cacheApis'; +import apiTokenApis from '../../controllers/apiTokenApis'; +import hookFilterApis from '../../controllers/hookFilterApis'; +import testApis from '../../controllers/testApis'; import { bulkDataAliasApis, dataAliasApis, @@ -41,20 +41,20 @@ import { dataAliasNestedApis, dataApis, oldDataApis, -} from './dataApis'; +} from '../../controllers/dataApis'; import { publicDataApis, publicDataExportApis, publicMetaApis, -} from './publicApis'; +} from '../../controllers/publicApis'; import { Server, Socket } from 'socket.io'; import passport from 'passport'; import crypto from 'crypto'; -import swaggerApis from './swagger/swaggerApis'; -import importApis from './sync/importApis'; -import syncSourceApis from './sync/syncSourceApis'; -import mapViewApis from './mapViewApis'; +import swaggerApis from '../../controllers/swagger/swaggerApis'; +import importApis from '../../controllers/sync/importApis'; +import syncSourceApis from '../../controllers/sync/syncSourceApis'; +import mapViewApis from '../../controllers/mapViewApis'; const clients: { [id: string]: Socket } = {}; const jobs: { [id: string]: { last_message: any } } = {}; From 60c08e0dba718204e5327d67138820b82cb66cca Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 23:55:05 +0530 Subject: [PATCH 02/64] refactor: rename controller files Signed-off-by: Pranav C --- ...{apiTokenApis.ts => apiTokenController.ts} | 0 ...achmentApis.ts => attachmentController.ts} | 0 .../{auditApis.ts => auditController.ts} | 0 .../{baseApis.ts => baseController.ts} | 0 .../{cacheApis.ts => cacheController.ts} | 0 .../{columnApis.ts => columnController.ts} | 0 .../{exportApis.ts => exportController.ts} | 0 .../{filterApis.ts => filterController.ts} | 0 ...umnApis.ts => formViewColumnController.ts} | 0 ...{formViewApis.ts => formViewController.ts} | 0 ...ryViewApis.ts => galleryViewController.ts} | 0 ...umnApis.ts => gridViewColumnController.ts} | 0 ...{gridViewApis.ts => gridViewController.ts} | 0 .../{hookApis.ts => hookController.ts} | 0 ...kFilterApis.ts => hookFilterController.ts} | 0 ...banViewApis.ts => kanbanViewController.ts} | 0 .../{mapViewApis.ts => mapViewController.ts} | 0 ...{metaDiffApis.ts => metaDiffController.ts} | 0 ...tyApis.ts => modelVisibilityController.ts} | 0 ...LicenseApis.ts => orgLicenseController.ts} | 0 ...{orgTokenApis.ts => orgTokenController.ts} | 0 .../{orgUserApis.ts => orgUserController.ts} | 2 +- .../{pluginApis.ts => pluginController.ts} | 0 .../{projectApis.ts => projectController.ts} | 0 ...ctUserApis.ts => projectUserController.ts} | 0 .../controllers/publicApis/publicDataApis.ts | 2 +- ...redBaseApis.ts => sharedBaseController.ts} | 0 .../{sortApis.ts => sortController.ts} | 0 .../{tableApis.ts => tableController.ts} | 2 +- .../{testApis.ts => testController.ts} | 0 .../{utilApis.ts => utilController.ts} | 0 ...wColumnApis.ts => viewColumnController.ts} | 0 .../{viewApis.ts => viewController.ts} | 2 +- .../src/lib/meta/api/helpers/populateMeta.ts | 2 +- packages/nocodb/src/lib/meta/api/index.ts | 64 +++++++++---------- 35 files changed, 37 insertions(+), 37 deletions(-) rename packages/nocodb/src/lib/controllers/{apiTokenApis.ts => apiTokenController.ts} (100%) rename packages/nocodb/src/lib/controllers/{attachmentApis.ts => attachmentController.ts} (100%) rename packages/nocodb/src/lib/controllers/{auditApis.ts => auditController.ts} (100%) rename packages/nocodb/src/lib/controllers/{baseApis.ts => baseController.ts} (100%) rename packages/nocodb/src/lib/controllers/{cacheApis.ts => cacheController.ts} (100%) rename packages/nocodb/src/lib/controllers/{columnApis.ts => columnController.ts} (100%) rename packages/nocodb/src/lib/controllers/{exportApis.ts => exportController.ts} (100%) rename packages/nocodb/src/lib/controllers/{filterApis.ts => filterController.ts} (100%) rename packages/nocodb/src/lib/controllers/{formViewColumnApis.ts => formViewColumnController.ts} (100%) rename packages/nocodb/src/lib/controllers/{formViewApis.ts => formViewController.ts} (100%) rename packages/nocodb/src/lib/controllers/{galleryViewApis.ts => galleryViewController.ts} (100%) rename packages/nocodb/src/lib/controllers/{gridViewColumnApis.ts => gridViewColumnController.ts} (100%) rename packages/nocodb/src/lib/controllers/{gridViewApis.ts => gridViewController.ts} (100%) rename packages/nocodb/src/lib/controllers/{hookApis.ts => hookController.ts} (100%) rename packages/nocodb/src/lib/controllers/{hookFilterApis.ts => hookFilterController.ts} (100%) rename packages/nocodb/src/lib/controllers/{kanbanViewApis.ts => kanbanViewController.ts} (100%) rename packages/nocodb/src/lib/controllers/{mapViewApis.ts => mapViewController.ts} (100%) rename packages/nocodb/src/lib/controllers/{metaDiffApis.ts => metaDiffController.ts} (100%) rename packages/nocodb/src/lib/controllers/{modelVisibilityApis.ts => modelVisibilityController.ts} (100%) rename packages/nocodb/src/lib/controllers/{orgLicenseApis.ts => orgLicenseController.ts} (100%) rename packages/nocodb/src/lib/controllers/{orgTokenApis.ts => orgTokenController.ts} (100%) rename packages/nocodb/src/lib/controllers/{orgUserApis.ts => orgUserController.ts} (99%) rename packages/nocodb/src/lib/controllers/{pluginApis.ts => pluginController.ts} (100%) rename packages/nocodb/src/lib/controllers/{projectApis.ts => projectController.ts} (100%) rename packages/nocodb/src/lib/controllers/{projectUserApis.ts => projectUserController.ts} (100%) rename packages/nocodb/src/lib/controllers/{sharedBaseApis.ts => sharedBaseController.ts} (100%) rename packages/nocodb/src/lib/controllers/{sortApis.ts => sortController.ts} (100%) rename packages/nocodb/src/lib/controllers/{tableApis.ts => tableController.ts} (99%) rename packages/nocodb/src/lib/controllers/{testApis.ts => testController.ts} (100%) rename packages/nocodb/src/lib/controllers/{utilApis.ts => utilController.ts} (100%) rename packages/nocodb/src/lib/controllers/{viewColumnApis.ts => viewColumnController.ts} (100%) rename packages/nocodb/src/lib/controllers/{viewApis.ts => viewController.ts} (98%) diff --git a/packages/nocodb/src/lib/controllers/apiTokenApis.ts b/packages/nocodb/src/lib/controllers/apiTokenController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/apiTokenApis.ts rename to packages/nocodb/src/lib/controllers/apiTokenController.ts diff --git a/packages/nocodb/src/lib/controllers/attachmentApis.ts b/packages/nocodb/src/lib/controllers/attachmentController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/attachmentApis.ts rename to packages/nocodb/src/lib/controllers/attachmentController.ts diff --git a/packages/nocodb/src/lib/controllers/auditApis.ts b/packages/nocodb/src/lib/controllers/auditController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/auditApis.ts rename to packages/nocodb/src/lib/controllers/auditController.ts diff --git a/packages/nocodb/src/lib/controllers/baseApis.ts b/packages/nocodb/src/lib/controllers/baseController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/baseApis.ts rename to packages/nocodb/src/lib/controllers/baseController.ts diff --git a/packages/nocodb/src/lib/controllers/cacheApis.ts b/packages/nocodb/src/lib/controllers/cacheController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/cacheApis.ts rename to packages/nocodb/src/lib/controllers/cacheController.ts diff --git a/packages/nocodb/src/lib/controllers/columnApis.ts b/packages/nocodb/src/lib/controllers/columnController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/columnApis.ts rename to packages/nocodb/src/lib/controllers/columnController.ts diff --git a/packages/nocodb/src/lib/controllers/exportApis.ts b/packages/nocodb/src/lib/controllers/exportController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/exportApis.ts rename to packages/nocodb/src/lib/controllers/exportController.ts diff --git a/packages/nocodb/src/lib/controllers/filterApis.ts b/packages/nocodb/src/lib/controllers/filterController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/filterApis.ts rename to packages/nocodb/src/lib/controllers/filterController.ts diff --git a/packages/nocodb/src/lib/controllers/formViewColumnApis.ts b/packages/nocodb/src/lib/controllers/formViewColumnController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/formViewColumnApis.ts rename to packages/nocodb/src/lib/controllers/formViewColumnController.ts diff --git a/packages/nocodb/src/lib/controllers/formViewApis.ts b/packages/nocodb/src/lib/controllers/formViewController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/formViewApis.ts rename to packages/nocodb/src/lib/controllers/formViewController.ts diff --git a/packages/nocodb/src/lib/controllers/galleryViewApis.ts b/packages/nocodb/src/lib/controllers/galleryViewController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/galleryViewApis.ts rename to packages/nocodb/src/lib/controllers/galleryViewController.ts diff --git a/packages/nocodb/src/lib/controllers/gridViewColumnApis.ts b/packages/nocodb/src/lib/controllers/gridViewColumnController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/gridViewColumnApis.ts rename to packages/nocodb/src/lib/controllers/gridViewColumnController.ts diff --git a/packages/nocodb/src/lib/controllers/gridViewApis.ts b/packages/nocodb/src/lib/controllers/gridViewController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/gridViewApis.ts rename to packages/nocodb/src/lib/controllers/gridViewController.ts diff --git a/packages/nocodb/src/lib/controllers/hookApis.ts b/packages/nocodb/src/lib/controllers/hookController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/hookApis.ts rename to packages/nocodb/src/lib/controllers/hookController.ts diff --git a/packages/nocodb/src/lib/controllers/hookFilterApis.ts b/packages/nocodb/src/lib/controllers/hookFilterController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/hookFilterApis.ts rename to packages/nocodb/src/lib/controllers/hookFilterController.ts diff --git a/packages/nocodb/src/lib/controllers/kanbanViewApis.ts b/packages/nocodb/src/lib/controllers/kanbanViewController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/kanbanViewApis.ts rename to packages/nocodb/src/lib/controllers/kanbanViewController.ts diff --git a/packages/nocodb/src/lib/controllers/mapViewApis.ts b/packages/nocodb/src/lib/controllers/mapViewController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/mapViewApis.ts rename to packages/nocodb/src/lib/controllers/mapViewController.ts diff --git a/packages/nocodb/src/lib/controllers/metaDiffApis.ts b/packages/nocodb/src/lib/controllers/metaDiffController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/metaDiffApis.ts rename to packages/nocodb/src/lib/controllers/metaDiffController.ts diff --git a/packages/nocodb/src/lib/controllers/modelVisibilityApis.ts b/packages/nocodb/src/lib/controllers/modelVisibilityController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/modelVisibilityApis.ts rename to packages/nocodb/src/lib/controllers/modelVisibilityController.ts diff --git a/packages/nocodb/src/lib/controllers/orgLicenseApis.ts b/packages/nocodb/src/lib/controllers/orgLicenseController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/orgLicenseApis.ts rename to packages/nocodb/src/lib/controllers/orgLicenseController.ts diff --git a/packages/nocodb/src/lib/controllers/orgTokenApis.ts b/packages/nocodb/src/lib/controllers/orgTokenController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/orgTokenApis.ts rename to packages/nocodb/src/lib/controllers/orgTokenController.ts diff --git a/packages/nocodb/src/lib/controllers/orgUserApis.ts b/packages/nocodb/src/lib/controllers/orgUserController.ts similarity index 99% rename from packages/nocodb/src/lib/controllers/orgUserApis.ts rename to packages/nocodb/src/lib/controllers/orgUserController.ts index 0dde26bce6..5da0bccee9 100644 --- a/packages/nocodb/src/lib/controllers/orgUserApis.ts +++ b/packages/nocodb/src/lib/controllers/orgUserController.ts @@ -23,7 +23,7 @@ import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import { randomTokenString } from '../meta/helpers/stringHelpers'; import { getAjvValidatorMw } from '../meta/api/helpers'; -import { sendInviteEmail } from './projectUserApis'; +import { sendInviteEmail } from './projectUserController'; async function userList(req, res) { res.json( diff --git a/packages/nocodb/src/lib/controllers/pluginApis.ts b/packages/nocodb/src/lib/controllers/pluginController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/pluginApis.ts rename to packages/nocodb/src/lib/controllers/pluginController.ts diff --git a/packages/nocodb/src/lib/controllers/projectApis.ts b/packages/nocodb/src/lib/controllers/projectController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/projectApis.ts rename to packages/nocodb/src/lib/controllers/projectController.ts diff --git a/packages/nocodb/src/lib/controllers/projectUserApis.ts b/packages/nocodb/src/lib/controllers/projectUserController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/projectUserApis.ts rename to packages/nocodb/src/lib/controllers/projectUserController.ts diff --git a/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts b/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts index cc81ef56a3..17e58cf1a5 100644 --- a/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts +++ b/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts @@ -15,7 +15,7 @@ import path from 'path'; import { nanoid } from 'nanoid'; import { mimeIcons } from '../../utils/mimeTypes'; import slash from 'slash'; -import { sanitizeUrlPath } from '../attachmentApis'; +import { sanitizeUrlPath } from '../attachmentController'; import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; import { getColumnByIdOrName } from '../dataApis/helpers'; import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants'; diff --git a/packages/nocodb/src/lib/controllers/sharedBaseApis.ts b/packages/nocodb/src/lib/controllers/sharedBaseController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/sharedBaseApis.ts rename to packages/nocodb/src/lib/controllers/sharedBaseController.ts diff --git a/packages/nocodb/src/lib/controllers/sortApis.ts b/packages/nocodb/src/lib/controllers/sortController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/sortApis.ts rename to packages/nocodb/src/lib/controllers/sortController.ts diff --git a/packages/nocodb/src/lib/controllers/tableApis.ts b/packages/nocodb/src/lib/controllers/tableController.ts similarity index 99% rename from packages/nocodb/src/lib/controllers/tableApis.ts rename to packages/nocodb/src/lib/controllers/tableController.ts index db15fac179..c4ca42d0b5 100644 --- a/packages/nocodb/src/lib/controllers/tableApis.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -19,7 +19,7 @@ import Project from '../models/Project'; import Audit from '../models/Audit'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { getAjvValidatorMw } from '../meta/api/helpers'; -import { xcVisibilityMetaGet } from './modelVisibilityApis'; +import { xcVisibilityMetaGet } from './modelVisibilityController'; import View from '../models/View'; import getColumnPropsFromUIDT from '../meta/helpers/getColumnPropsFromUIDT'; import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; diff --git a/packages/nocodb/src/lib/controllers/testApis.ts b/packages/nocodb/src/lib/controllers/testController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/testApis.ts rename to packages/nocodb/src/lib/controllers/testController.ts diff --git a/packages/nocodb/src/lib/controllers/utilApis.ts b/packages/nocodb/src/lib/controllers/utilController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/utilApis.ts rename to packages/nocodb/src/lib/controllers/utilController.ts diff --git a/packages/nocodb/src/lib/controllers/viewColumnApis.ts b/packages/nocodb/src/lib/controllers/viewColumnController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/viewColumnApis.ts rename to packages/nocodb/src/lib/controllers/viewColumnController.ts diff --git a/packages/nocodb/src/lib/controllers/viewApis.ts b/packages/nocodb/src/lib/controllers/viewController.ts similarity index 98% rename from packages/nocodb/src/lib/controllers/viewApis.ts rename to packages/nocodb/src/lib/controllers/viewController.ts index da8e387878..4cc86a2d54 100644 --- a/packages/nocodb/src/lib/controllers/viewApis.ts +++ b/packages/nocodb/src/lib/controllers/viewController.ts @@ -12,7 +12,7 @@ import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; import Project from '../models/Project'; import View from '../models/View'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { xcVisibilityMetaGet } from './modelVisibilityApis'; +import { xcVisibilityMetaGet } from './modelVisibilityController'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; // @ts-ignore export async function viewGet(req: Request, res: Response
) {} diff --git a/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts b/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts index 7f4e7bc41f..b4a7098da0 100644 --- a/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts +++ b/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts @@ -11,7 +11,7 @@ import getTableNameAlias, { import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; import getColumnUiType from '../../helpers/getColumnUiType'; import mapDefaultDisplayValue from '../../helpers/mapDefaultDisplayValue'; -import { extractAndGenerateManyToManyRelations } from '../../../controllers/metaDiffApis'; +import { extractAndGenerateManyToManyRelations } from '../../../controllers/metaDiffController'; import { ModelTypes, UITypes, ViewTypes } from 'nocodb-sdk'; import { IGNORE_TABLES } from '../../../utils/common/BaseApiBuilder'; diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index c7285c7330..06da16fd10 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -1,39 +1,39 @@ import { Tele } from 'nc-help'; -import orgLicenseApis from '../../controllers/orgLicenseApis'; -import orgTokenApis from '../../controllers/orgTokenApis'; -import orgUserApis from '../../controllers/orgUserApis'; -import projectApis from '../../controllers/projectApis'; -import baseApis from '../../controllers/baseApis'; -import tableApis from '../../controllers/tableApis'; -import columnApis from '../../controllers/columnApis'; +import orgLicenseApis from '../../controllers/orgLicenseController'; +import orgTokenApis from '../../controllers/orgTokenController'; +import orgUserApis from '../../controllers/orgUserController'; +import projectApis from '../../controllers/projectController'; +import baseApis from '../../controllers/baseController'; +import tableApis from '../../controllers/tableController'; +import columnApis from '../../controllers/columnController'; import { Router } from 'express'; -import sortApis from '../../controllers/sortApis'; -import filterApis from '../../controllers/filterApis'; -import viewColumnApis from '../../controllers/viewColumnApis'; -import gridViewApis from '../../controllers/gridViewApis'; -import viewApis from '../../controllers/viewApis'; -import galleryViewApis from '../../controllers/galleryViewApis'; -import formViewApis from '../../controllers/formViewApis'; -import formViewColumnApis from '../../controllers/formViewColumnApis'; -import attachmentApis from '../../controllers/attachmentApis'; -import exportApis from '../../controllers/exportApis'; -import auditApis from '../../controllers/auditApis'; -import hookApis from '../../controllers/hookApis'; -import pluginApis from '../../controllers/pluginApis'; -import gridViewColumnApis from '../../controllers/gridViewColumnApis'; -import kanbanViewApis from '../../controllers/kanbanViewApis'; +import sortApis from '../../controllers/sortController'; +import filterApis from '../../controllers/filterController'; +import viewColumnApis from '../../controllers/viewColumnController'; +import gridViewApis from '../../controllers/gridViewController'; +import viewApis from '../../controllers/viewController'; +import galleryViewApis from '../../controllers/galleryViewController'; +import formViewApis from '../../controllers/formViewController'; +import formViewColumnApis from '../../controllers/formViewColumnController'; +import attachmentApis from '../../controllers/attachmentController'; +import exportApis from '../../controllers/exportController'; +import auditApis from '../../controllers/auditController'; +import hookApis from '../../controllers/hookController'; +import pluginApis from '../../controllers/pluginController'; +import gridViewColumnApis from '../../controllers/gridViewColumnController'; +import kanbanViewApis from '../../controllers/kanbanViewController'; import { userApis } from '../../controllers/userApi'; // import extractProjectIdAndAuthenticate from './helpers/extractProjectIdAndAuthenticate'; -import utilApis from '../../controllers/utilApis'; -import projectUserApis from '../../controllers/projectUserApis'; -import sharedBaseApis from '../../controllers/sharedBaseApis'; +import utilApis from '../../controllers/utilController'; +import projectUserApis from '../../controllers/projectUserController'; +import sharedBaseApis from '../../controllers/sharedBaseController'; import { initStrategies } from '../../controllers/userApi/initStrategies'; -import modelVisibilityApis from '../../controllers/modelVisibilityApis'; -import metaDiffApis from '../../controllers/metaDiffApis'; -import cacheApis from '../../controllers/cacheApis'; -import apiTokenApis from '../../controllers/apiTokenApis'; -import hookFilterApis from '../../controllers/hookFilterApis'; -import testApis from '../../controllers/testApis'; +import modelVisibilityApis from '../../controllers/modelVisibilityController'; +import metaDiffApis from '../../controllers/metaDiffController'; +import cacheApis from '../../controllers/cacheController'; +import apiTokenApis from '../../controllers/apiTokenController'; +import hookFilterApis from '../../controllers/hookFilterController'; +import testApis from '../../controllers/testController'; import { bulkDataAliasApis, dataAliasApis, @@ -54,7 +54,7 @@ import crypto from 'crypto'; import swaggerApis from '../../controllers/swagger/swaggerApis'; import importApis from '../../controllers/sync/importApis'; import syncSourceApis from '../../controllers/sync/syncSourceApis'; -import mapViewApis from '../../controllers/mapViewApis'; +import mapViewApis from '../../controllers/mapViewController'; const clients: { [id: string]: Socket } = {}; const jobs: { [id: string]: { last_message: any } } = {}; From 9c9ceb7cd8770f32743681bf0c7e905c4881db1f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sun, 26 Feb 2023 14:24:09 +0530 Subject: [PATCH 03/64] feat(wip): controller-service implementation Signed-off-by: Pranav C --- packages/nc-gui/components.d.ts | 1 + .../src/lib/controllers/tableController.ts | 410 ++---------------- packages/nocodb/src/lib/models/ProjectUser.ts | 2 +- packages/nocodb/src/lib/services/index.ts | 2 + .../nocodb/src/lib/services/projectService.ts | 42 ++ .../nocodb/src/lib/services/tableService.ts | 348 +++++++++++++++ 6 files changed, 428 insertions(+), 377 deletions(-) create mode 100644 packages/nocodb/src/lib/services/index.ts create mode 100644 packages/nocodb/src/lib/services/projectService.ts create mode 100644 packages/nocodb/src/lib/services/tableService.ts diff --git a/packages/nc-gui/components.d.ts b/packages/nc-gui/components.d.ts index 78f37aba7f..372758da9a 100644 --- a/packages/nc-gui/components.d.ts +++ b/packages/nc-gui/components.d.ts @@ -89,6 +89,7 @@ declare module '@vue/runtime-core' { IcTwotoneWidthNormal: typeof import('~icons/ic/twotone-width-normal')['default'] LogosGoogleGmail: typeof import('~icons/logos/google-gmail')['default'] LogosMysqlIcon: typeof import('~icons/logos/mysql-icon')['default'] + LogosOracle: typeof import('~icons/logos/oracle')['default'] LogosPostgresql: typeof import('~icons/logos/postgresql')['default'] LogosRedditIcon: typeof import('~icons/logos/reddit-icon')['default'] LogosSnowflakeIcon: typeof import('~icons/logos/snowflake-icon')['default'] diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index c4ca42d0b5..728cc1816a 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -1,404 +1,62 @@ import { Request, Response, Router } from 'express'; -import Model from '../models/Model'; -import { Tele } from 'nc-help'; -import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import DOMPurify from 'isomorphic-dompurify'; -import { - AuditOperationSubTypes, - AuditOperationTypes, - isVirtualCol, - ModelTypes, - NormalColumnRequestType, - TableListType, - TableReqType, - TableType, - UITypes, -} from 'nocodb-sdk'; -import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; -import Project from '../models/Project'; -import Audit from '../models/Audit'; -import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { TableListType, TableReqType, TableType } from 'nocodb-sdk'; import { getAjvValidatorMw } from '../meta/api/helpers'; -import { xcVisibilityMetaGet } from './modelVisibilityController'; -import View from '../models/View'; -import getColumnPropsFromUIDT from '../meta/helpers/getColumnPropsFromUIDT'; -import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; -import { NcError } from '../meta/helpers/catchError'; -import getTableNameAlias, { getColumnNameAlias } from '../meta/helpers/getTableName'; -import Column from '../models/Column'; -import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; -import getColumnUiType from '../meta/helpers/getColumnUiType'; -import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; +import { tableUpdate } from '../meta/api/tableApis'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -export async function tableGet(req: Request, res: Response) { - const table = await Model.getWithInfo({ - id: req.params.tableId, - }); - - // todo: optimise - const viewList = await xcVisibilityMetaGet(table.project_id, [table]); - - //await View.list(req.params.tableId) - table.views = viewList.filter((table: any) => { - return Object.keys((req as any).session?.passport?.user?.roles).some( - (role) => - (req as any)?.session?.passport?.user?.roles[role] && - !table.disabled[role] - ); - }); - - res.json(table); -} - -export async function tableReorder(req: Request, res: Response) { - res.json(Model.updateOrder(req.params.tableId, req.body.order)); -} +import { tableService } from '../services'; export async function tableList(req: Request, res: Response) { - const viewList = await xcVisibilityMetaGet(req.params.projectId); - - // todo: optimise - const tableViewMapping = viewList.reduce((o, view: any) => { - o[view.fk_model_id] = o[view.fk_model_id] || 0; - if ( - Object.keys((req as any).session?.passport?.user?.roles).some( - (role) => - (req as any)?.session?.passport?.user?.roles[role] && - !view.disabled[role] - ) - ) { - o[view.fk_model_id]++; - } - return o; - }, {}); - - const tableList = ( - await Model.list({ - project_id: req.params.projectId, - base_id: req.params.baseId, - }) - ).filter((t) => tableViewMapping[t.id]); - res.json( new PagedResponseImpl( - req.query?.includeM2M === 'true' - ? tableList - : (tableList.filter((t) => !t.mm) as Model[]) + await tableService.getAccessibleTables({ + projectId: req.params.projectId, + baseId: req.params.baseId, + includeM2M: req.query?.includeM2M === 'true', + roles: (req as any).session?.passport?.user?.roles, + }) ) ); } export async function tableCreate(req: Request, res) { - const project = await Project.getWithInfo(req.params.projectId); - let base = project.bases[0]; - - if (req.params.baseId) { - base = project.bases.find((b) => b.id === req.params.baseId); - } - - if ( - !req.body.table_name || - (project.prefix && project.prefix === req.body.table_name) - ) { - NcError.badRequest( - 'Missing table name `table_name` property in request body' - ); - } - - if (base.is_meta && project.prefix) { - if (!req.body.table_name.startsWith(project.prefix)) { - req.body.table_name = `${project.prefix}_${req.body.table_name}`; - } - } - - req.body.table_name = DOMPurify.sanitize(req.body.table_name); - - // validate table name - if (/^\s+|\s+$/.test(req.body.table_name)) { - NcError.badRequest( - 'Leading or trailing whitespace not allowed in table names' - ); - } - - if ( - !(await Model.checkTitleAvailable({ - table_name: req.body.table_name, - project_id: project.id, - base_id: base.id, - })) - ) { - NcError.badRequest('Duplicate table name'); - } - - if (!req.body.title) { - req.body.title = getTableNameAlias( - req.body.table_name, - project.prefix, - base - ); - } - - if ( - !(await Model.checkAliasAvailable({ - title: req.body.title, - project_id: project.id, - base_id: base.id, - })) - ) { - NcError.badRequest('Duplicate table alias'); - } - - const sqlMgr = await ProjectMgrv2.getSqlMgr(project); - - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - - let tableNameLengthLimit = 255; - const sqlClientType = sqlClient.knex.clientType(); - if (sqlClientType === 'mysql2' || sqlClientType === 'mysql') { - tableNameLengthLimit = 64; - } else if (sqlClientType === 'pg') { - tableNameLengthLimit = 63; - } else if (sqlClientType === 'mssql') { - tableNameLengthLimit = 128; - } - - if (req.body.table_name.length > tableNameLengthLimit) { - NcError.badRequest(`Table name exceeds ${tableNameLengthLimit} characters`); - } - - const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType); - - for (const column of req.body.columns) { - if (column.column_name.length > mxColumnLength) { - NcError.badRequest( - `Column name ${column.column_name} exceeds ${mxColumnLength} characters` - ); - } - } - - req.body.columns = req.body.columns?.map((c) => ({ - ...getColumnPropsFromUIDT(c as any, base), - cn: c.column_name, - })); - await sqlMgr.sqlOpPlus(base, 'tableCreate', { - ...req.body, - tn: req.body.table_name, + const result = tableService.createTable({ + projectId: req.params.projectId, + baseId: req.params.baseId, + table: req.body, + user: (req as any).session?.passport?.user, }); - const columns: Array< - Omit & { - cn: string; - system?: boolean; - } - > = (await sqlClient.columnList({ tn: req.body.table_name }))?.data?.list; - - const tables = await Model.list({ - project_id: project.id, - base_id: base.id, - }); - - await Audit.insert({ - project_id: project.id, - base_id: base.id, - op_type: AuditOperationTypes.TABLE, - op_sub_type: AuditOperationSubTypes.CREATED, - user: (req as any)?.user?.email, - description: `created table ${req.body.table_name} with alias ${req.body.title} `, - ip: (req as any).clientIp, - }).then(() => {}); - - mapDefaultDisplayValue(req.body.columns); - - Tele.emit('evt', { evt_type: 'table:created' }); - - res.json( - await Model.insert(project.id, base.id, { - ...req.body, - columns: columns.map((c, i) => { - const colMetaFromReq = req.body?.columns?.find( - (c1) => c.cn === c1.column_name - ) as NormalColumnRequestType; - return { - ...colMetaFromReq, - uidt: - (colMetaFromReq?.uidt as string) || - c.uidt || - getColumnUiType(base, c), - ...c, - dtxp: [UITypes.MultiSelect, UITypes.SingleSelect].includes( - colMetaFromReq.uidt as any - ) - ? colMetaFromReq.dtxp - : c.dtxp, - title: colMetaFromReq?.title || getColumnNameAlias(c.cn, base), - column_name: c.cn, - order: i + 1, - } as NormalColumnRequestType; - }), - order: +(tables?.pop()?.order ?? 0) + 1, - }) - ); + res.json(result); } -export async function tableUpdate(req: Request, res) { - const model = await Model.get(req.params.tableId); - - const project = await Project.getWithInfo( - req.body.project_id || (req as any).ncProjectId - ); - const base = project.bases.find((b) => b.id === model.base_id); - - if (model.project_id !== project.id) { - NcError.badRequest('Model does not belong to project'); - } - - // if meta present update meta and return - // todo: allow user to update meta and other prop in single api call - if ('meta' in req.body) { - await Model.updateMeta(req.params.tableId, req.body.meta); - - return res.json({ msg: 'success' }); - } - - if (!req.body.table_name) { - NcError.badRequest( - 'Missing table name `table_name` property in request body' - ); - } - - if (base.is_meta && project.prefix) { - if (!req.body.table_name.startsWith(project.prefix)) { - req.body.table_name = `${project.prefix}${req.body.table_name}`; - } - } - - req.body.table_name = DOMPurify.sanitize(req.body.table_name); - - // validate table name - if (/^\s+|\s+$/.test(req.body.table_name)) { - NcError.badRequest( - 'Leading or trailing whitespace not allowed in table names' - ); - } - - if ( - !(await Model.checkTitleAvailable({ - table_name: req.body.table_name, - project_id: project.id, - base_id: base.id, - })) - ) { - NcError.badRequest('Duplicate table name'); - } - - if (!req.body.title) { - req.body.title = getTableNameAlias( - req.body.table_name, - project.prefix, - base - ); - } - - if ( - !(await Model.checkAliasAvailable({ - title: req.body.title, - project_id: project.id, - base_id: base.id, - })) - ) { - NcError.badRequest('Duplicate table alias'); - } - - const sqlMgr = await ProjectMgrv2.getSqlMgr(project); - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - - let tableNameLengthLimit = 255; - const sqlClientType = sqlClient.knex.clientType(); - if (sqlClientType === 'mysql2' || sqlClientType === 'mysql') { - tableNameLengthLimit = 64; - } else if (sqlClientType === 'pg') { - tableNameLengthLimit = 63; - } else if (sqlClientType === 'mssql') { - tableNameLengthLimit = 128; - } - - if (req.body.table_name.length > tableNameLengthLimit) { - NcError.badRequest(`Table name exceeds ${tableNameLengthLimit} characters`); - } - - await Model.updateAliasAndTableName( - req.params.tableId, - req.body.title, - req.body.table_name - ); - - await sqlMgr.sqlOpPlus(base, 'tableRename', { - ...req.body, - tn: req.body.table_name, - tn_old: model.table_name, +export async function tableGet(req: Request, res: Response) { + const table = await tableService.getTableWithAccessibleViews({ + tableId: req.params.tableId, + user: (req as any).session?.passport?.user, }); - Tele.emit('evt', { evt_type: 'table:updated' }); - - res.json({ msg: 'success' }); + res.json(table); } export async function tableDelete(req: Request, res: Response) { - const table = await Model.getByIdOrName({ id: req.params.tableId }); - await table.getColumns(); - - const relationColumns = table.columns.filter( - (c) => c.uidt === UITypes.LinkToAnotherRecord - ); - - if (relationColumns?.length) { - const referredTables = await Promise.all( - relationColumns.map(async (c) => - c - .getColOptions() - .then((opt) => opt.getRelatedTable()) - .then() - ) - ); - NcError.badRequest( - `Table can't be deleted since Table is being referred in following tables : ${referredTables.join( - ', ' - )}. Delete LinkToAnotherRecord columns and try again.` - ); - } - - const project = await Project.getWithInfo(table.project_id); - const base = project.bases.find((b) => b.id === table.base_id); - const sqlMgr = await ProjectMgrv2.getSqlMgr(project); - (table as any).tn = table.table_name; - table.columns = table.columns.filter((c) => !isVirtualCol(c)); - table.columns.forEach((c) => { - (c as any).cn = c.column_name; + const result = await tableService.deleteTable({ + tableId: req.params.tableId, + user: (req as any).session?.passport?.user, }); - if (table.type === ModelTypes.TABLE) { - await sqlMgr.sqlOpPlus(base, 'tableDelete', table); - } else if (table.type === ModelTypes.VIEW) { - await sqlMgr.sqlOpPlus(base, 'viewDelete', { - ...table, - view_name: table.table_name, - }); - } - - await Audit.insert({ - project_id: project.id, - base_id: base.id, - op_type: AuditOperationTypes.TABLE, - op_sub_type: AuditOperationSubTypes.DELETED, - user: (req as any)?.user?.email, - description: `Deleted ${table.type} ${table.table_name} with alias ${table.title} `, - ip: (req as any).clientIp, - }).then(() => {}); - - Tele.emit('evt', { evt_type: 'table:deleted' }); + res.json(result); +} - res.json(await table.delete()); +export async function tableReorder(req: Request, res: Response) { + res.json( + await tableService.reorderTable({ + tableId: req.params.tableId, + order: req.body.order, + }) + ); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/models/ProjectUser.ts b/packages/nocodb/src/lib/models/ProjectUser.ts index d62f5490e8..d701853b2d 100644 --- a/packages/nocodb/src/lib/models/ProjectUser.ts +++ b/packages/nocodb/src/lib/models/ProjectUser.ts @@ -219,7 +219,7 @@ export default class ProjectUser { static async getProjectsList( userId: string, - _params: any, + _params?: any, ncMeta = Noco.ncMeta ): Promise { // todo: pagination diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts new file mode 100644 index 0000000000..afe5d99b06 --- /dev/null +++ b/packages/nocodb/src/lib/services/index.ts @@ -0,0 +1,2 @@ +// export * as projectService from './projectService'; +export * as tableService from './tableService'; diff --git a/packages/nocodb/src/lib/services/projectService.ts b/packages/nocodb/src/lib/services/projectService.ts new file mode 100644 index 0000000000..1520f6de21 --- /dev/null +++ b/packages/nocodb/src/lib/services/projectService.ts @@ -0,0 +1,42 @@ +// // services/project.service.ts +// import { OrgUserRoles, ProjectType } from 'nocodb-sdk' +// import Project from '../models/Project' +// import ProjectUser from '../models/ProjectUser' +// import { customAlphabet } from 'nanoid' +// import { NcError } from './helpers/catchError' +// +// const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 4) +// +// export async function getProjectWithInfo(projectId: string) { +// const project = await Project.getWithInfo(projectId) +// return project +// } +// +// export function sanitizeProject(project: any) { +// const sanitizedProject = { ...project } +// sanitizedProject.bases?.forEach((b: any) => { +// ['config'].forEach((k) => delete b[k]) +// }) +// return sanitizedProject +// } +// +// export async function updateProject(projectId: string, data: Partial) { +// const project = await Project.getWithInfo(projectId) +// if (data?.title && project.title !== data.title && (await Project.getByTitle(data.title))) { +// NcError.badRequest('Project title already in use') +// } +// const result = await Project.update(projectId, data) +// return result +// } +// +// export async function listProjects(user: any) { +// const projects = user?.roles?.includes(OrgUserRoles.SUPER_ADMIN) +// ? await Project.list() +// : await ProjectUser.getProjectsList(user.id) +// return projects as ProjectType[] +// } +// +// export async function softDeleteProject(projectId: string) { +// const result = await Project.softDelete(projectId) +// return result +// } diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts new file mode 100644 index 0000000000..2bf2c4ca9f --- /dev/null +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -0,0 +1,348 @@ +// @ts-ignore +import DOMPurify from 'isomorphic-dompurify'; +import { + AuditOperationSubTypes, + AuditOperationTypes, + isVirtualCol, + ModelTypes, + NormalColumnRequestType, + TableReqType, + UITypes, +} from 'nocodb-sdk'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; +import { NcError } from '../meta/helpers/catchError'; +import getColumnPropsFromUIDT from '../meta/helpers/getColumnPropsFromUIDT'; +import getColumnUiType from '../meta/helpers/getColumnUiType'; +import getTableNameAlias, { + getColumnNameAlias, +} from '../meta/helpers/getTableName'; +import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; +import Audit from '../models/Audit'; +import Column from '../models/Column'; +import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; +import Model from '../models/Model'; +import ModelRoleVisibility from '../models/ModelRoleVisibility'; +import Project from '../models/Project'; +import User from '../models/User'; +import View from '../models/View'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import { Tele } from 'nc-help'; + +export function reorderTable(param: { tableId: string; order: any }) { + return Model.updateOrder(param.tableId, param.order); +} + +export async function deleteTable(param: { tableId: string; user: User }) { + const table = await Model.getByIdOrName({ id: param.tableId }); + await table.getColumns(); + + const relationColumns = table.columns.filter( + (c) => c.uidt === UITypes.LinkToAnotherRecord + ); + + if (relationColumns?.length) { + const referredTables = await Promise.all( + relationColumns.map(async (c) => + c + .getColOptions() + .then((opt) => opt.getRelatedTable()) + .then() + ) + ); + NcError.badRequest( + `Table can't be deleted since Table is being referred in following tables : ${referredTables.join( + ', ' + )}. Delete LinkToAnotherRecord columns and try again.` + ); + } + + const project = await Project.getWithInfo(table.project_id); + const base = project.bases.find((b) => b.id === table.base_id); + const sqlMgr = await ProjectMgrv2.getSqlMgr(project); + (table as any).tn = table.table_name; + table.columns = table.columns.filter((c) => !isVirtualCol(c)); + table.columns.forEach((c) => { + (c as any).cn = c.column_name; + }); + + if (table.type === ModelTypes.TABLE) { + await sqlMgr.sqlOpPlus(base, 'tableDelete', table); + } else if (table.type === ModelTypes.VIEW) { + await sqlMgr.sqlOpPlus(base, 'viewDelete', { + ...table, + view_name: table.table_name, + }); + } + + await Audit.insert({ + project_id: project.id, + base_id: base.id, + op_type: AuditOperationTypes.TABLE, + op_sub_type: AuditOperationSubTypes.DELETED, + user: param.user?.email, + description: `Deleted ${table.type} ${table.table_name} with alias ${table.title} `, + // ip: (req as any).clientIp, + }).then(() => {}); + + Tele.emit('evt', { evt_type: 'table:deleted' }); + + return table.delete(); +} + +export async function getTableWithAccessibleViews(param: { + tableId: string; + user: User; +}) { + const table = await Model.getWithInfo({ + id: param.tableId, + }); + + // todo: optimise + const viewList = await xcVisibilityMetaGet(table.project_id, [table]); + + //await View.list(req.params.tableId) + table.views = viewList.filter((table: any) => { + return Object.keys(param.user?.roles).some( + (role) => param.user?.roles[role] && !table.disabled[role] + ); + }); + + return table; +} + +export async function xcVisibilityMetaGet( + projectId, + _models: Model[] = null, + includeM2M = true + // type: 'table' | 'tableAndViews' | 'views' = 'table' +) { + // todo: move to + const roles = ['owner', 'creator', 'viewer', 'editor', 'commenter', 'guest']; + + const defaultDisabled = roles.reduce((o, r) => ({ ...o, [r]: false }), {}); + + let models = + _models || + (await Model.list({ + project_id: projectId, + base_id: undefined, + })); + + models = includeM2M ? models : (models.filter((t) => !t.mm) as Model[]); + + const result = await models.reduce(async (_obj, model) => { + const obj = await _obj; + + const views = await model.getViews(); + for (const view of views) { + obj[view.id] = { + ptn: model.table_name, + _ptn: model.title, + ptype: model.type, + tn: view.title, + _tn: view.title, + table_meta: model.meta, + ...view, + disabled: { ...defaultDisabled }, + }; + } + + return obj; + }, Promise.resolve({})); + + const disabledList = await ModelRoleVisibility.list(projectId); + + for (const d of disabledList) { + if (result[d.fk_view_id]) + result[d.fk_view_id].disabled[d.role] = !!d.disabled; + } + + return Object.values(result); +} + +export async function getAccessibleTables(param: { + projectId: string; + baseId: string; + includeM2M?: boolean; + roles: Record; +}) { + const viewList = await xcVisibilityMetaGet(param.projectId); + + // todo: optimise + const tableViewMapping = viewList.reduce((o, view: any) => { + o[view.fk_model_id] = o[view.fk_model_id] || 0; + if ( + Object.keys(param.roles).some( + (role) => param.roles[role] && !view.disabled[role] + ) + ) { + o[view.fk_model_id]++; + } + return o; + }, {}); + + const tableList = ( + await Model.list({ + project_id: param.projectId, + base_id: param.baseId, + }) + ).filter((t) => tableViewMapping[t.id]); + + return param.includeM2M + ? tableList + : (tableList.filter((t) => !t.mm) as Model[]); +} + +export async function createTable(args: { + projectId: string; + baseId?: string; + table: TableReqType; + user: User; +}) { + const project = await Project.getWithInfo(args.projectId); + let base = project.bases[0]; + + if (args.baseId) { + base = project.bases.find((b) => b.id === args.baseId); + } + + if ( + !args.table.table_name || + (project.prefix && project.prefix === args.table.table_name) + ) { + NcError.badRequest( + 'Missing table name `table_name` property in request body' + ); + } + + if (base.is_meta && project.prefix) { + if (!args.table.table_name.startsWith(project.prefix)) { + args.table.table_name = `${project.prefix}_${args.table.table_name}`; + } + } + + args.table.table_name = DOMPurify.sanitize(args.table.table_name); + + // validate table name + if (/^\s+|\s+$/.test(args.table.table_name)) { + NcError.badRequest( + 'Leading or trailing whitespace not allowed in table names' + ); + } + + if ( + !(await Model.checkTitleAvailable({ + table_name: args.table.table_name, + project_id: project.id, + base_id: base.id, + })) + ) { + NcError.badRequest('Duplicate table name'); + } + + if (!args.table.title) { + args.table.title = getTableNameAlias( + args.table.table_name, + project.prefix, + base + ); + } + + if ( + !(await Model.checkAliasAvailable({ + title: args.table.title, + project_id: project.id, + base_id: base.id, + })) + ) { + NcError.badRequest('Duplicate table alias'); + } + + const sqlMgr = await ProjectMgrv2.getSqlMgr(project); + + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + + let tableNameLengthLimit = 255; + const sqlClientType = sqlClient.knex.clientType(); + if (sqlClientType === 'mysql2' || sqlClientType === 'mysql') { + tableNameLengthLimit = 64; + } else if (sqlClientType === 'pg') { + tableNameLengthLimit = 63; + } else if (sqlClientType === 'mssql') { + tableNameLengthLimit = 128; + } + + if (args.table.table_name.length > tableNameLengthLimit) { + NcError.badRequest(`Table name exceeds ${tableNameLengthLimit} characters`); + } + + const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType); + + for (const column of args.table.columns) { + if (column.column_name.length > mxColumnLength) { + NcError.badRequest( + `Column name ${column.column_name} exceeds ${mxColumnLength} characters` + ); + } + } + + args.table.columns = args.table.columns?.map((c) => ({ + ...getColumnPropsFromUIDT(c as any, base), + cn: c.column_name, + })); + await sqlMgr.sqlOpPlus(base, 'tableCreate', { + ...args.table, + tn: args.table.table_name, + }); + + const columns: Array< + Omit & { + cn: string; + system?: boolean; + } + > = (await sqlClient.columnList({ tn: args.table.table_name }))?.data?.list; + + const tables = await Model.list({ + project_id: project.id, + base_id: base.id, + }); + + await Audit.insert({ + project_id: project.id, + base_id: base.id, + op_type: AuditOperationTypes.TABLE, + op_sub_type: AuditOperationSubTypes.CREATED, + user: args.user?.email, + description: `created table ${args.table.table_name} with alias ${args.table.title} `, + // ip: (req as any).clientIp, + }).then(() => {}); + + mapDefaultDisplayValue(args.table.columns); + + Tele.emit('evt', { evt_type: 'table:created' }); + + const result = await Model.insert(project.id, base.id, { + ...args.table, + columns: columns.map((c, i) => { + const colMetaFromReq = args.table?.columns?.find( + (c1) => c.cn === c1.column_name + ); + return { + ...colMetaFromReq, + uidt: colMetaFromReq?.uidt || c.uidt || getColumnUiType(base, c), + ...c, + dtxp: [UITypes.MultiSelect, UITypes.SingleSelect].includes( + colMetaFromReq.uidt as any + ) + ? colMetaFromReq.dtxp + : c.dtxp, + title: colMetaFromReq?.title || getColumnNameAlias(c.cn, base), + column_name: c.cn, + order: i + 1, + } as NormalColumnRequestType; + }), + order: +(tables?.pop()?.order ?? 0) + 1, + }); + + return result; +} From 60507cc43363e5661d44762414febb535b735c12 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 27 Feb 2023 12:51:57 +0530 Subject: [PATCH 04/64] refactor: handler to service-controller - table and column Signed-off-by: Pranav C --- packages/nocodb-sdk/src/lib/sqlUi/OracleUi.ts | 6 +- .../nocodb-sdk/src/lib/sqlUi/SqlUiFactory.ts | 11 +- .../src/lib/controllers/columnController.ts | 1784 +---------------- .../src/lib/controllers/filterController.ts | 24 +- .../src/lib/meta/api/helpers/columnHelpers.ts | 8 +- .../nocodb/src/lib/services/columnService.ts | 1778 ++++++++++++++++ .../nocodb/src/lib/services/filterService.ts | 11 + packages/nocodb/src/lib/services/index.ts | 2 + .../nocodb/src/lib/services/tableService.ts | 4 +- 9 files changed, 1836 insertions(+), 1792 deletions(-) create mode 100644 packages/nocodb/src/lib/services/columnService.ts create mode 100644 packages/nocodb/src/lib/services/filterService.ts diff --git a/packages/nocodb-sdk/src/lib/sqlUi/OracleUi.ts b/packages/nocodb-sdk/src/lib/sqlUi/OracleUi.ts index aa157d07d5..3498a0835e 100644 --- a/packages/nocodb-sdk/src/lib/sqlUi/OracleUi.ts +++ b/packages/nocodb-sdk/src/lib/sqlUi/OracleUi.ts @@ -1,3 +1,4 @@ +import { NormalColumnRequestType } from '../Api' import UITypes from '../UITypes'; import { IDType } from './index'; @@ -794,7 +795,10 @@ export class OracleUi { } } - static getDataTypeForUiType(col: { uidt: UITypes }, idType?: IDType) { + static getDataTypeForUiType( + col: { uidt: UITypes | NormalColumnRequestType['uidt'] }, + idType?: IDType + ) { const colProp: any = {}; switch (col.uidt) { case 'ID': diff --git a/packages/nocodb-sdk/src/lib/sqlUi/SqlUiFactory.ts b/packages/nocodb-sdk/src/lib/sqlUi/SqlUiFactory.ts index 6e79066cad..3eef12622a 100644 --- a/packages/nocodb-sdk/src/lib/sqlUi/SqlUiFactory.ts +++ b/packages/nocodb-sdk/src/lib/sqlUi/SqlUiFactory.ts @@ -1,3 +1,4 @@ +import { BoolType } from '../Api' import UITypes from '../UITypes'; import { MssqlUi } from './MssqlUi'; @@ -56,12 +57,12 @@ export type SqlUIColumn = { dt?: string; dtx?: string; ct?: string; - nrqd?: boolean; - rqd?: boolean; + nrqd?: BoolType; + rqd?: BoolType; ck?: string; - pk?: boolean; - un?: boolean; - ai?: boolean; + pk?: BoolType; + un?: BoolType; + ai?: BoolType; cdf?: string | any; clen?: number | any; np?: string; diff --git a/packages/nocodb/src/lib/controllers/columnController.ts b/packages/nocodb/src/lib/controllers/columnController.ts index 78aa196dcb..dad4a4c01b 100644 --- a/packages/nocodb/src/lib/controllers/columnController.ts +++ b/packages/nocodb/src/lib/controllers/columnController.ts @@ -1,1785 +1,45 @@ -import { Request, Response, Router } from 'express'; -import Model from '../models/Model'; -import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; -import Base from '../models/Base'; -import Column from '../models/Column'; -import { Tele } from 'nc-help'; -import validateParams from '../meta/helpers/validateParams'; - -import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; -import { - getUniqueColumnAliasName, - getUniqueColumnName, -} from '../meta/helpers/getUniqueName'; -import { - AuditOperationSubTypes, - AuditOperationTypes, - ColumnReqType, - isVirtualCol, - LinkToAnotherColumnReqType, - LinkToAnotherRecordType, - RelationTypes, - substituteColumnAliasWithIdInFormula, - substituteColumnIdWithAliasInFormula, - TableType, - UITypes, -} from 'nocodb-sdk'; -import Audit from '../models/Audit'; -import SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2'; -import Noco from '../Noco'; -import NcMetaIO from '../meta/NcMetaIO'; -import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { NcError } from '../meta/helpers/catchError'; -import getColumnPropsFromUIDT from '../meta/helpers/getColumnPropsFromUIDT'; -import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; -import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; -import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import FormulaColumn from '../models/FormulaColumn'; -import KanbanView from '../models/KanbanView'; -import { MetaTable } from '../utils/globals'; -import formulaQueryBuilderv2 from '../db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2'; -import { - createHmAndBtColumn, - generateFkName, - getAjvValidatorMw, - randomID, - validateLookupPayload, - validateRequiredField, - validateRollupPayload, -} from '../meta/api/helpers'; - -export enum Altered { - NEW_COLUMN = 1, - DELETE_COLUMN = 4, - UPDATE_COLUMN = 8, -} +import { Request, Response, Router } from 'express' +import { ColumnReqType, TableType, UITypes } from 'nocodb-sdk'; +import { getAjvValidatorMw } from '../meta/api/helpers' +import { metaApiMetrics } from '../meta/helpers/apiMetrics' +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw' +import { columnService } from '../services'; export async function columnGet(req: Request, res: Response) { - res.json(await Column.get({ colId: req.params.columnId })); + res.json(await columnService.columnGet({ columnId: req.params.columnId })); } export async function columnAdd( req: Request, res: Response ) { - const table = await Model.getWithInfo({ - id: req.params.tableId, - }); - - const base = await Base.get(table.base_id); - - const project = await base.getProject(); - - if (req.body.title || req.body.column_name) { - const dbDriver = NcConnectionMgrv2.get(base); - - const sqlClientType = dbDriver.clientType(); - - const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType); - - if ((req.body.title || req.body.column_name).length > mxColumnLength) { - NcError.badRequest( - `Column name ${ - req.body.title || req.body.column_name - } exceeds ${mxColumnLength} characters` - ); - } - } - - if ( - !isVirtualCol(req.body) && - !(await Column.checkTitleAvailable({ - column_name: req.body.column_name, - fk_model_id: req.params.tableId, - })) - ) { - NcError.badRequest('Duplicate column name'); - } - if ( - !(await Column.checkAliasAvailable({ - title: req.body.title || req.body.column_name, - fk_model_id: req.params.tableId, - })) - ) { - NcError.badRequest('Duplicate column alias'); - } - - let colBody: any = req.body; - switch (colBody.uidt) { - case UITypes.Rollup: - { - await validateRollupPayload(req.body); - - await Column.insert({ - ...colBody, - fk_model_id: table.id, - }); - } - break; - case UITypes.Lookup: - { - await validateLookupPayload(req.body); - - await Column.insert({ - ...colBody, - fk_model_id: table.id, - }); - } - break; - - case UITypes.LinkToAnotherRecord: - { - validateParams(['parentId', 'childId', 'type'], req.body); - - // get parent and child models - const parent = await Model.getWithInfo({ - id: (req.body as LinkToAnotherColumnReqType).parentId, - }); - const child = await Model.getWithInfo({ - id: (req.body as LinkToAnotherColumnReqType).childId, - }); - let childColumn: Column; - - const sqlMgr = await ProjectMgrv2.getSqlMgr({ - id: base.project_id, - }); - if ( - (req.body as LinkToAnotherColumnReqType).type === 'hm' || - (req.body as LinkToAnotherColumnReqType).type === 'bt' - ) { - // populate fk column name - const fkColName = getUniqueColumnName( - await child.getColumns(), - `${parent.table_name}_id` - ); - - let foreignKeyName; - { - // create foreign key - const newColumn = { - cn: fkColName, - - title: fkColName, - column_name: fkColName, - rqd: false, - pk: false, - ai: false, - cdf: null, - dt: parent.primaryKey.dt, - dtxp: parent.primaryKey.dtxp, - dtxs: parent.primaryKey.dtxs, - un: parent.primaryKey.un, - altered: Altered.NEW_COLUMN, - }; - const tableUpdateBody = { - ...child, - tn: child.table_name, - originalColumns: child.columns.map((c) => ({ - ...c, - cn: c.column_name, - })), - columns: [ - ...child.columns.map((c) => ({ - ...c, - cn: c.column_name, - })), - newColumn, - ], - }; - - await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); - - const { id } = await Column.insert({ - ...newColumn, - uidt: UITypes.ForeignKey, - fk_model_id: child.id, - }); - - childColumn = await Column.get({ colId: id }); - - // ignore relation creation if virtual - if (!(req.body as LinkToAnotherColumnReqType).virtual) { - foreignKeyName = generateFkName(parent, child); - // create relation - await sqlMgr.sqlOpPlus(base, 'relationCreate', { - childColumn: fkColName, - childTable: child.table_name, - parentTable: parent.table_name, - onDelete: 'NO ACTION', - onUpdate: 'NO ACTION', - type: 'real', - parentColumn: parent.primaryKey.column_name, - foreignKeyName, - }); - } - - // todo: create index for virtual relations as well - // create index for foreign key in pg - if (base.type === 'pg') { - await createColumnIndex({ - column: new Column({ - ...newColumn, - fk_model_id: child.id, - }), - base, - sqlMgr, - }); - } - } - await createHmAndBtColumn( - child, - parent, - childColumn, - (req.body as LinkToAnotherColumnReqType).type as RelationTypes, - (req.body as LinkToAnotherColumnReqType).title, - foreignKeyName, - (req.body as LinkToAnotherColumnReqType).virtual - ); - } else if ((req.body as LinkToAnotherColumnReqType).type === 'mm') { - const aTn = `${project?.prefix ?? ''}_nc_m2m_${randomID()}`; - const aTnAlias = aTn; - - const parentPK = parent.primaryKey; - const childPK = child.primaryKey; - - const associateTableCols = []; - - const parentCn = 'table1_id'; - const childCn = 'table2_id'; - - associateTableCols.push( - { - cn: childCn, - column_name: childCn, - title: childCn, - rqd: true, - pk: true, - ai: false, - cdf: null, - dt: childPK.dt, - dtxp: childPK.dtxp, - dtxs: childPK.dtxs, - un: childPK.un, - altered: 1, - uidt: UITypes.ForeignKey, - }, - { - cn: parentCn, - column_name: parentCn, - title: parentCn, - rqd: true, - pk: true, - ai: false, - cdf: null, - dt: parentPK.dt, - dtxp: parentPK.dtxp, - dtxs: parentPK.dtxs, - un: parentPK.un, - altered: 1, - uidt: UITypes.ForeignKey, - } - ); - - await sqlMgr.sqlOpPlus(base, 'tableCreate', { - tn: aTn, - _tn: aTnAlias, - columns: associateTableCols, - }); - - const assocModel = await Model.insert(project.id, base.id, { - table_name: aTn, - title: aTnAlias, - // todo: sanitize - mm: true, - columns: associateTableCols, - }); - - let foreignKeyName1; - let foreignKeyName2; - - if (!(req.body as LinkToAnotherColumnReqType).virtual) { - foreignKeyName1 = generateFkName(parent, child); - foreignKeyName2 = generateFkName(parent, child); - - const rel1Args = { - ...req.body, - childTable: aTn, - childColumn: parentCn, - parentTable: parent.table_name, - parentColumn: parentPK.column_name, - type: 'real', - foreignKeyName: foreignKeyName1, - }; - const rel2Args = { - ...req.body, - childTable: aTn, - childColumn: childCn, - parentTable: child.table_name, - parentColumn: childPK.column_name, - type: 'real', - foreignKeyName: foreignKeyName2, - }; - - await sqlMgr.sqlOpPlus(base, 'relationCreate', rel1Args); - await sqlMgr.sqlOpPlus(base, 'relationCreate', rel2Args); - } - const parentCol = (await assocModel.getColumns())?.find( - (c) => c.column_name === parentCn - ); - const childCol = (await assocModel.getColumns())?.find( - (c) => c.column_name === childCn - ); - - await createHmAndBtColumn( - assocModel, - child, - childCol, - null, - null, - foreignKeyName1, - (req.body as LinkToAnotherColumnReqType).virtual, - true - ); - await createHmAndBtColumn( - assocModel, - parent, - parentCol, - null, - null, - foreignKeyName2, - (req.body as LinkToAnotherColumnReqType).virtual, - true - ); - - await Column.insert({ - title: getUniqueColumnAliasName( - await child.getColumns(), - `${parent.title} List` - ), - uidt: UITypes.LinkToAnotherRecord, - type: 'mm', - - // ref_db_alias - fk_model_id: child.id, - // db_type: - - fk_child_column_id: childPK.id, - fk_parent_column_id: parentPK.id, - - fk_mm_model_id: assocModel.id, - fk_mm_child_column_id: childCol.id, - fk_mm_parent_column_id: parentCol.id, - fk_related_model_id: parent.id, - }); - await Column.insert({ - title: getUniqueColumnAliasName( - await parent.getColumns(), - req.body.title ?? `${child.title} List` - ), - - uidt: UITypes.LinkToAnotherRecord, - type: 'mm', - - fk_model_id: parent.id, - - fk_child_column_id: parentPK.id, - fk_parent_column_id: childPK.id, - - fk_mm_model_id: assocModel.id, - fk_mm_child_column_id: parentCol.id, - fk_mm_parent_column_id: childCol.id, - fk_related_model_id: child.id, - }); - - // todo: create index for virtual relations as well - // create index for foreign key in pg - if (base.type === 'pg') { - await createColumnIndex({ - column: new Column({ - ...associateTableCols[0], - fk_model_id: assocModel.id, - }), - base, - sqlMgr, - }); - await createColumnIndex({ - column: new Column({ - ...associateTableCols[1], - fk_model_id: assocModel.id, - }), - base, - sqlMgr, - }); - } - } - } - Tele.emit('evt', { evt_type: 'relation:created' }); - break; - - case UITypes.QrCode: - await Column.insert({ - ...colBody, - fk_model_id: table.id, - }); - break; - case UITypes.Barcode: - await Column.insert({ - ...colBody, - fk_model_id: table.id, - }); - break; - case UITypes.Formula: - colBody.formula = await substituteColumnAliasWithIdInFormula( - colBody.formula_raw || colBody.formula, - table.columns - ); - - try { - // test the query to see if it is valid in db level - const dbDriver = NcConnectionMgrv2.get(base); - await formulaQueryBuilderv2(colBody.formula, null, dbDriver, table); - } catch (e) { - console.error(e); - NcError.badRequest('Invalid Formula'); - } - - await Column.insert({ - ...colBody, - fk_model_id: table.id, - }); - - break; - default: - { - colBody = getColumnPropsFromUIDT(colBody, base); - if (colBody.uidt === UITypes.Duration) { - colBody.dtxp = '20'; - // by default, colBody.dtxs is 2 - // Duration column needs more that that - colBody.dtxs = '4'; - } - - if ( - [UITypes.SingleSelect, UITypes.MultiSelect].includes(colBody.uidt) - ) { - const dbDriver = NcConnectionMgrv2.get(base); - const driverType = dbDriver.clientType(); - const optionTitles = colBody.colOptions.options.map((el) => - el.title.replace(/'/g, "''") - ); - - // this is not used for select columns and cause issue for MySQL - colBody.dtxs = ''; - // Handle default values - if (colBody.cdf) { - if (colBody.uidt === UITypes.SingleSelect) { - if (!optionTitles.includes(colBody.cdf.replace(/'/g, "''"))) { - NcError.badRequest( - `Default value '${colBody.cdf}' is not a select option.` - ); - } - } else { - for (const cdf of colBody.cdf.split(',')) { - if (!optionTitles.includes(cdf.replace(/'/g, "''"))) { - NcError.badRequest( - `Default value '${cdf}' is not a select option.` - ); - } - } - } - - // handle single quote for default value - if (driverType === 'mysql' || driverType === 'mysql2') { - colBody.cdf = colBody.cdf.replace(/'/g, "'"); - } else { - colBody.cdf = colBody.cdf.replace(/'/g, "''"); - } - - if (driverType === 'pg') { - colBody.cdf = `'${colBody.cdf}'`; - } - } - - // Restrict duplicates - const titles = colBody.colOptions.options.map((el) => el.title); - if ( - titles.some(function (item) { - return titles.indexOf(item) !== titles.lastIndexOf(item); - }) - ) { - NcError.badRequest('Duplicates are not allowed!'); - } - - // Restrict empty options - if ( - titles.some(function (item) { - return item === ''; - }) - ) { - NcError.badRequest('Empty options are not allowed!'); - } - - // Trim end of enum/set - if (colBody.dt === 'enum' || colBody.dt === 'set') { - for (const opt of colBody.colOptions.options) { - opt.title = opt.title.trimEnd(); - } - } - - if (colBody.uidt === UITypes.SingleSelect) { - colBody.dtxp = colBody.colOptions?.options.length - ? `${colBody.colOptions.options - .map((o) => `'${o.title.replace(/'/gi, "''")}'`) - .join(',')}` - : ''; - } else if (colBody.uidt === UITypes.MultiSelect) { - colBody.dtxp = colBody.colOptions?.options.length - ? `${colBody.colOptions.options - .map((o) => { - if (o.title.includes(',')) { - NcError.badRequest("Illegal char(',') for MultiSelect"); - } - return `'${o.title.replace(/'/gi, "''")}'`; - }) - .join(',')}` - : ''; - } - - // Handle empty enum/set for mysql (we restrict empty user options beforehand) - if (driverType === 'mysql' || driverType === 'mysql2') { - if ( - !colBody.colOptions.options.length && - (!colBody.dtxp || colBody.dtxp === '') - ) { - colBody.dtxp = "''"; - } - - if (colBody.dt === 'set') { - if (colBody.colOptions?.options.length > 64) { - colBody.dt = 'text'; - } - } - } - } - - const tableUpdateBody = { - ...table, - tn: table.table_name, - originalColumns: table.columns.map((c) => ({ - ...c, - cn: c.column_name, - })), - columns: [ - ...table.columns.map((c) => ({ ...c, cn: c.column_name })), - { - ...colBody, - cn: colBody.column_name, - altered: Altered.NEW_COLUMN, - }, - ], - }; - - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: base.project_id }); - await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); - - const columns: Array< - Omit & { - cn: string; - system?: boolean; - } - > = (await sqlClient.columnList({ tn: table.table_name }))?.data?.list; - - const insertedColumnMeta = - columns.find((c) => c.cn === colBody.column_name) || ({} as any); - - await Column.insert({ - ...colBody, - ...insertedColumnMeta, - dtxp: [UITypes.MultiSelect, UITypes.SingleSelect].includes( - colBody.uidt as any - ) - ? colBody.dtxp - : insertedColumnMeta.dtxp, - fk_model_id: table.id, - }); - } - break; - } - - await table.getColumns(); - - await Audit.insert({ - project_id: base.project_id, - op_type: AuditOperationTypes.TABLE_COLUMN, - op_sub_type: AuditOperationSubTypes.CREATED, - user: (req as any)?.user?.email, - description: `created column ${colBody.column_name} with alias ${colBody.title} from table ${table.table_name}`, - ip: (req as any).clientIp, - }).then(() => {}); - - Tele.emit('evt', { evt_type: 'column:created' }); - - res.json(table); + res.json( + await columnService.columnAdd({ + tableId: req.params.tableId, + column: req.body, + }) + ); } export async function columnSetAsPrimary(req: Request, res: Response) { - const column = await Column.get({ colId: req.params.columnId }); - res.json(await Model.updatePrimaryColumn(column.fk_model_id, column.id)); -} - -async function updateRollupOrLookup(colBody: any, column: Column) { - if ( - UITypes.Lookup === column.uidt && - validateRequiredField(colBody, [ - 'fk_lookup_column_id', - 'fk_relation_column_id', - ]) - ) { - await validateLookupPayload(colBody, column.id); - await Column.update(column.id, colBody); - } else if ( - UITypes.Rollup === column.uidt && - validateRequiredField(colBody, [ - 'fk_relation_column_id', - 'fk_rollup_column_id', - 'rollup_function', - ]) - ) { - await validateRollupPayload(colBody); - await Column.update(column.id, colBody); - } + res.json( + await columnService.columnSetAsPrimary({ columnId: req.params.columnId }) + ); } -export async function columnUpdate(req: Request, res: Response) { - const column = await Column.get({ colId: req.params.columnId }); - - const table = await Model.getWithInfo({ - id: column.fk_model_id, - }); - const base = await Base.get(table.base_id); - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - - const sqlClientType = sqlClient.knex.clientType(); - - const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType); - - if (req.body.column_name.length > mxColumnLength) { - NcError.badRequest( - `Column name ${req.body.column_name} exceeds ${mxColumnLength} characters` - ); - } - - if ( - !isVirtualCol(req.body) && - !(await Column.checkTitleAvailable({ - column_name: req.body.column_name, - fk_model_id: column.fk_model_id, - exclude_id: req.params.columnId, - })) - ) { - NcError.badRequest('Duplicate column name'); - } - if ( - !(await Column.checkAliasAvailable({ - title: req.body.title, - fk_model_id: column.fk_model_id, - exclude_id: req.params.columnId, - })) - ) { - NcError.badRequest('Duplicate column alias'); - } - - let colBody = req.body; - if ( - [ - UITypes.Lookup, - UITypes.Rollup, - UITypes.LinkToAnotherRecord, - UITypes.Formula, - UITypes.QrCode, - UITypes.Barcode, - UITypes.ForeignKey, - ].includes(column.uidt) - ) { - if (column.uidt === colBody.uidt) { - if ([UITypes.QrCode, UITypes.Barcode].includes(column.uidt)) { - await Column.update(column.id, { - ...column, - ...colBody, - }); - } else if (column.uidt === UITypes.Formula) { - colBody.formula = await substituteColumnAliasWithIdInFormula( - colBody.formula_raw || colBody.formula, - table.columns - ); - - try { - // test the query to see if it is valid in db level - const dbDriver = NcConnectionMgrv2.get(base); - await formulaQueryBuilderv2(colBody.formula, null, dbDriver, table); - } catch (e) { - console.error(e); - NcError.badRequest('Invalid Formula'); - } - - await Column.update(column.id, { - // title: colBody.title, - ...column, - ...colBody, - }); - } else if (colBody.title !== column.title) { - await Column.updateAlias(req.params.columnId, { - title: colBody.title, - }); - } - await updateRollupOrLookup(colBody, column); - } else { - NcError.notImplemented( - `Updating ${colBody.uidt} => ${colBody.uidt} is not implemented` - ); - } - } else if ( - [ - UITypes.Lookup, - UITypes.Rollup, - UITypes.LinkToAnotherRecord, - UITypes.Formula, - UITypes.QrCode, - UITypes.Barcode, - UITypes.ForeignKey, - ].includes(colBody.uidt) - ) { - NcError.notImplemented( - `Updating ${colBody.uidt} => ${colBody.uidt} is not implemented` - ); - } else if ( - [UITypes.SingleSelect, UITypes.MultiSelect].includes(colBody.uidt) - ) { - colBody = getColumnPropsFromUIDT(colBody, base); - - const baseModel = await Model.getBaseModelSQL({ - id: table.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - if (colBody.colOptions?.options) { - const supportedDrivers = ['mysql', 'mysql2', 'pg', 'mssql', 'sqlite3']; - const dbDriver = NcConnectionMgrv2.get(base); - const driverType = dbDriver.clientType(); - - // MultiSelect to SingleSelect - if ( - column.uidt === UITypes.MultiSelect && - colBody.uidt === UITypes.SingleSelect - ) { - if (driverType === 'mysql' || driverType === 'mysql2') { - await dbDriver.raw( - `UPDATE ?? SET ?? = SUBSTRING_INDEX(??, ',', 1) WHERE ?? LIKE '%,%';`, - [ - table.table_name, - column.column_name, - column.column_name, - column.column_name, - ] - ); - } else if (driverType === 'pg') { - await dbDriver.raw(`UPDATE ?? SET ?? = split_part(??, ',', 1);`, [ - table.table_name, - column.column_name, - column.column_name, - ]); - } else if (driverType === 'mssql') { - await dbDriver.raw( - `UPDATE ?? SET ?? = LEFT(cast(?? as varchar(max)), CHARINDEX(',', ??) - 1) WHERE CHARINDEX(',', ??) > 0;`, - [ - table.table_name, - column.column_name, - column.column_name, - column.column_name, - column.column_name, - ] - ); - } else if (driverType === 'sqlite3') { - await dbDriver.raw( - `UPDATE ?? SET ?? = substr(??, 1, instr(??, ',') - 1) WHERE ?? LIKE '%,%';`, - [ - table.table_name, - column.column_name, - column.column_name, - column.column_name, - column.column_name, - ] - ); - } - } - - // Handle migrations - if (column.colOptions?.options) { - for (const op of column.colOptions.options.filter( - (el) => el.order === null - )) { - op.title = op.title.replace(/^'/, '').replace(/'$/, ''); - } - } - - // Handle default values - const optionTitles = colBody.colOptions.options.map((el) => - el.title.replace(/'/g, "''") - ); - if (colBody.cdf) { - if (colBody.uidt === UITypes.SingleSelect) { - if (!optionTitles.includes(colBody.cdf.replace(/'/g, "''"))) { - NcError.badRequest( - `Default value '${colBody.cdf}' is not a select option.` - ); - } - } else { - for (const cdf of colBody.cdf.split(',')) { - if (!optionTitles.includes(cdf.replace(/'/g, "''"))) { - NcError.badRequest( - `Default value '${cdf}' is not a select option.` - ); - } - } - } - - // handle single quote for default value - if (driverType === 'mysql' || driverType === 'mysql2') { - colBody.cdf = colBody.cdf.replace(/'/g, "'"); - } else { - colBody.cdf = colBody.cdf.replace(/'/g, "''"); - } - - if (driverType === 'pg') { - colBody.cdf = `'${colBody.cdf}'`; - } - } - - // Restrict duplicates - const titles = colBody.colOptions.options.map((el) => el.title); - if ( - titles.some(function (item) { - return titles.indexOf(item) !== titles.lastIndexOf(item); - }) - ) { - NcError.badRequest('Duplicates are not allowed!'); - } - - // Restrict empty options - if ( - titles.some(function (item) { - return item === ''; - }) - ) { - NcError.badRequest('Empty options are not allowed!'); - } - - // Trim end of enum/set - if (colBody.dt === 'enum' || colBody.dt === 'set') { - for (const opt of colBody.colOptions.options) { - opt.title = opt.title.trimEnd(); - } - } - - if (colBody.uidt === UITypes.SingleSelect) { - colBody.dtxp = colBody.colOptions?.options.length - ? `${colBody.colOptions.options - .map((o) => `'${o.title.replace(/'/gi, "''")}'`) - .join(',')}` - : ''; - } else if (colBody.uidt === UITypes.MultiSelect) { - colBody.dtxp = colBody.colOptions?.options.length - ? `${colBody.colOptions.options - .map((o) => { - if (o.title.includes(',')) { - NcError.badRequest("Illegal char(',') for MultiSelect"); - } - return `'${o.title.replace(/'/gi, "''")}'`; - }) - .join(',')}` - : ''; - } - - // Handle empty enum/set for mysql (we restrict empty user options beforehand) - if (driverType === 'mysql' || driverType === 'mysql2') { - if ( - !colBody.colOptions.options.length && - (!colBody.dtxp || colBody.dtxp === '') - ) { - colBody.dtxp = "''"; - } - - if (colBody.dt === 'set') { - if (colBody.colOptions?.options.length > 64) { - colBody.dt = 'text'; - } - } - } - - // Handle option delete - if (column.colOptions?.options) { - for (const option of column.colOptions.options.filter((oldOp) => - colBody.colOptions.options.find((newOp) => newOp.id === oldOp.id) - ? false - : true - )) { - if ( - !supportedDrivers.includes(driverType) && - column.uidt === UITypes.MultiSelect - ) { - NcError.badRequest( - 'Your database not yet supported for this operation. Please remove option from records manually before dropping.' - ); - } - if (column.uidt === UITypes.SingleSelect) { - if (driverType === 'mssql') { - await dbDriver.raw(`UPDATE ?? SET ?? = NULL WHERE ?? LIKE ?`, [ - table.table_name, - column.column_name, - column.column_name, - option.title, - ]); - } else { - await baseModel.bulkUpdateAll( - { where: `(${column.title},eq,${option.title})` }, - { [column.column_name]: null }, - { cookie: req } - ); - } - } else if (column.uidt === UITypes.MultiSelect) { - if (driverType === 'mysql' || driverType === 'mysql2') { - if (colBody.dt === 'set') { - await dbDriver.raw( - `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), ',')) WHERE FIND_IN_SET(?, ??)`, - [ - table.table_name, - column.column_name, - column.column_name, - option.title, - option.title, - column.column_name, - ] - ); - } else { - await dbDriver.raw( - `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), ','))`, - [ - table.table_name, - column.column_name, - column.column_name, - option.title, - ] - ); - } - } else if (driverType === 'pg') { - await dbDriver.raw( - `UPDATE ?? SET ?? = array_to_string(array_remove(string_to_array(??, ','), ?), ',')`, - [ - table.table_name, - column.column_name, - column.column_name, - option.title, - ] - ); - } else if (driverType === 'mssql') { - await dbDriver.raw( - `UPDATE ?? SET ?? = substring(replace(concat(',', ??, ','), concat(',', ?, ','), ','), 2, len(replace(concat(',', ??, ','), concat(',', ?, ','), ',')) - 2)`, - [ - table.table_name, - column.column_name, - column.column_name, - option.title, - column.column_name, - option.title, - ] - ); - } else if (driverType === 'sqlite3') { - await dbDriver.raw( - `UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ','), ',')`, - [ - table.table_name, - column.column_name, - column.column_name, - option.title, - ] - ); - } - } - } - } - - const interchange = []; - - // Handle option update - if (column.colOptions?.options) { - const old_titles = column.colOptions.options.map((el) => el.title); - for (const option of column.colOptions.options.filter((oldOp) => - colBody.colOptions.options.find( - (newOp) => newOp.id === oldOp.id && newOp.title !== oldOp.title - ) - )) { - if ( - !supportedDrivers.includes(driverType) && - column.uidt === UITypes.MultiSelect - ) { - NcError.badRequest( - 'Your database not yet supported for this operation. Please remove option from records manually before updating.' - ); - } - - const newOp = { - ...colBody.colOptions.options.find((el) => option.id === el.id), - }; - if (old_titles.includes(newOp.title)) { - const def_option = { ...newOp }; - let title_counter = 1; - while (old_titles.includes(newOp.title)) { - newOp.title = `${def_option.title}_${title_counter++}`; - } - interchange.push({ - def_option, - temp_title: newOp.title, - }); - } - - // Append new option before editing - if ( - (driverType === 'mysql' || driverType === 'mysql2') && - (column.dt === 'enum' || column.dt === 'set') - ) { - column.colOptions.options.push({ title: newOp.title }); - - let temp_dtxp = ''; - if (column.uidt === UITypes.SingleSelect) { - temp_dtxp = column.colOptions.options.length - ? `${column.colOptions.options - .map((o) => `'${o.title.replace(/'/gi, "''")}'`) - .join(',')}` - : ''; - } else if (column.uidt === UITypes.MultiSelect) { - temp_dtxp = column.colOptions.options.length - ? `${column.colOptions.options - .map((o) => { - if (o.title.includes(',')) { - NcError.badRequest("Illegal char(',') for MultiSelect"); - throw new Error(''); - } - return `'${o.title.replace(/'/gi, "''")}'`; - }) - .join(',')}` - : ''; - } - - const tableUpdateBody = { - ...table, - tn: table.table_name, - originalColumns: table.columns.map((c) => ({ - ...c, - cn: c.column_name, - cno: c.column_name, - })), - columns: await Promise.all( - table.columns.map(async (c) => { - if (c.id === req.params.columnId) { - const res = { - ...c, - ...column, - cn: column.column_name, - cno: c.column_name, - dtxp: temp_dtxp, - altered: Altered.UPDATE_COLUMN, - }; - return Promise.resolve(res); - } else { - (c as any).cn = c.column_name; - } - return Promise.resolve(c); - }) - ), - }; - - const sqlMgr = await ProjectMgrv2.getSqlMgr({ - id: base.project_id, - }); - await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); - - await Column.update(req.params.columnId, { - ...column, - }); - } - - if (column.uidt === UITypes.SingleSelect) { - if (driverType === 'mssql') { - await dbDriver.raw(`UPDATE ?? SET ?? = ? WHERE ?? LIKE ?`, [ - table.table_name, - column.column_name, - newOp.title, - column.column_name, - option.title, - ]); - } else { - await baseModel.bulkUpdateAll( - { where: `(${column.title},eq,${option.title})` }, - { [column.column_name]: newOp.title }, - { cookie: req } - ); - } - } else if (column.uidt === UITypes.MultiSelect) { - if (driverType === 'mysql' || driverType === 'mysql2') { - if (colBody.dt === 'set') { - await dbDriver.raw( - `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), CONCAT(',', ?, ','))) WHERE FIND_IN_SET(?, ??)`, - [ - table.table_name, - column.column_name, - column.column_name, - option.title, - newOp.title, - option.title, - column.column_name, - ] - ); - } else { - await dbDriver.raw( - `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), CONCAT(',', ?, ',')))`, - [ - table.table_name, - column.column_name, - column.column_name, - option.title, - newOp.title, - ] - ); - } - } else if (driverType === 'pg') { - await dbDriver.raw( - `UPDATE ?? SET ?? = array_to_string(array_replace(string_to_array(??, ','), ?, ?), ',')`, - [ - table.table_name, - column.column_name, - column.column_name, - option.title, - newOp.title, - ] - ); - } else if (driverType === 'mssql') { - await dbDriver.raw( - `UPDATE ?? SET ?? = substring(replace(concat(',', ??, ','), concat(',', ?, ','), concat(',', ?, ',')), 2, len(replace(concat(',', ??, ','), concat(',', ?, ','), concat(',', ?, ','))) - 2)`, - [ - table.table_name, - column.column_name, - column.column_name, - option.title, - newOp.title, - column.column_name, - option.title, - newOp.title, - ] - ); - } else if (driverType === 'sqlite3') { - await dbDriver.raw( - `UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ',' || ? || ','), ',')`, - [ - table.table_name, - column.column_name, - column.column_name, - option.title, - newOp.title, - ] - ); - } - } - } - } - - for (const ch of interchange) { - const newOp = ch.def_option; - if (column.uidt === UITypes.SingleSelect) { - if (driverType === 'mssql') { - await dbDriver.raw(`UPDATE ?? SET ?? = ? WHERE ?? LIKE ?`, [ - table.table_name, - column.column_name, - newOp.title, - column.column_name, - ch.temp_title, - ]); - } else { - await baseModel.bulkUpdateAll( - { where: `(${column.title},eq,${ch.temp_title})` }, - { [column.column_name]: newOp.title }, - { cookie: req } - ); - } - } else if (column.uidt === UITypes.MultiSelect) { - if (driverType === 'mysql' || driverType === 'mysql2') { - if (colBody.dt === 'set') { - await dbDriver.raw( - `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), CONCAT(',', ?, ','))) WHERE FIND_IN_SET(?, ??)`, - [ - table.table_name, - column.column_name, - column.column_name, - ch.temp_title, - newOp.title, - ch.temp_title, - column.column_name, - ] - ); - } else { - await dbDriver.raw( - `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), CONCAT(',', ?, ',')))`, - [ - table.table_name, - column.column_name, - column.column_name, - ch.temp_title, - newOp.title, - ch.temp_title, - column.column_name, - ] - ); - } - } else if (driverType === 'pg') { - await dbDriver.raw( - `UPDATE ?? SET ?? = array_to_string(array_replace(string_to_array(??, ','), ?, ?), ',')`, - [ - table.table_name, - column.column_name, - column.column_name, - ch.temp_title, - newOp.title, - ] - ); - } else if (driverType === 'mssql') { - await dbDriver.raw( - `UPDATE ?? SET ?? = substring(replace(concat(',', ??, ','), concat(',', ?, ','), concat(',', ?, ',')), 2, len(replace(concat(',', ??, ','), concat(',', ?, ','), concat(',', ?, ','))) - 2)`, - [ - table.table_name, - column.column_name, - column.column_name, - ch.temp_title, - newOp.title, - column.column_name, - ch.temp_title, - newOp.title, - ] - ); - } else if (driverType === 'sqlite3') { - await dbDriver.raw( - `UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ',' || ? || ','), ',')`, - [ - table.table_name, - column.column_name, - column.column_name, - ch.temp_title, - newOp.title, - ] - ); - } - } - } - } - - const tableUpdateBody = { - ...table, - tn: table.table_name, - originalColumns: table.columns.map((c) => ({ - ...c, - cn: c.column_name, - cno: c.column_name, - })), - columns: await Promise.all( - table.columns.map(async (c) => { - if (c.id === req.params.columnId) { - const res = { - ...c, - ...colBody, - cn: colBody.column_name, - cno: c.column_name, - altered: Altered.UPDATE_COLUMN, - }; - - // update formula with new column name - if (c.column_name != colBody.column_name) { - const formulas = await Noco.ncMeta - .knex(MetaTable.COL_FORMULA) - .where('formula', 'like', `%${c.id}%`); - if (formulas) { - const new_column = c; - new_column.column_name = colBody.column_name; - new_column.title = colBody.title; - for (const f of formulas) { - // the formula with column IDs only - const formula = f.formula; - // replace column IDs with alias to get the new formula_raw - const new_formula_raw = substituteColumnIdWithAliasInFormula( - formula, - [new_column] - ); - await FormulaColumn.update(c.id, { - formula_raw: new_formula_raw, - }); - } - } - } - return Promise.resolve(res); - } else { - (c as any).cn = c.column_name; - } - return Promise.resolve(c); - }) - ), - }; - - const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: base.project_id }); - await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); - - await Column.update(req.params.columnId, { - ...colBody, - }); - } else { - colBody = getColumnPropsFromUIDT(colBody, base); - const tableUpdateBody = { - ...table, - tn: table.table_name, - originalColumns: table.columns.map((c) => ({ - ...c, - cn: c.column_name, - cno: c.column_name, - })), - columns: await Promise.all( - table.columns.map(async (c) => { - if (c.id === req.params.columnId) { - const res = { - ...c, - ...colBody, - cn: colBody.column_name, - cno: c.column_name, - altered: Altered.UPDATE_COLUMN, - }; - - // update formula with new column name - if (c.column_name != colBody.column_name) { - const formulas = await Noco.ncMeta - .knex(MetaTable.COL_FORMULA) - .where('formula', 'like', `%${c.id}%`); - if (formulas) { - const new_column = c; - new_column.column_name = colBody.column_name; - new_column.title = colBody.title; - for (const f of formulas) { - // the formula with column IDs only - const formula = f.formula; - // replace column IDs with alias to get the new formula_raw - const new_formula_raw = substituteColumnIdWithAliasInFormula( - formula, - [new_column] - ); - await FormulaColumn.update(c.id, { - formula_raw: new_formula_raw, - }); - } - } - } - return Promise.resolve(res); - } else { - (c as any).cn = c.column_name; - } - return Promise.resolve(c); - }) - ), - }; - - const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: base.project_id }); - await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); - - await Column.update(req.params.columnId, { - ...colBody, - }); - } - await Audit.insert({ - project_id: base.project_id, - op_type: AuditOperationTypes.TABLE_COLUMN, - op_sub_type: AuditOperationSubTypes.UPDATED, - user: (req as any)?.user?.email, - description: `updated column ${column.column_name} with alias ${column.title} from table ${table.table_name}`, - ip: (req as any).clientIp, - }).then(() => {}); - - await table.getColumns(); - Tele.emit('evt', { evt_type: 'column:updated' }); - - res.json(table); +export async function columnUpdate(req: Request, res: Response){ + res.json(await columnService.columnUpdate({ + columnId: req.params.columnId, + column: req.body + }) } export async function columnDelete(req: Request, res: Response) { - const column = await Column.get({ colId: req.params.columnId }); - const table = await Model.getWithInfo({ - id: column.fk_model_id, - }); - const base = await Base.get(table.base_id); - - // const ncMeta = await Noco.ncMeta.startTransaction(); - // const sql-mgr = await ProjectMgrv2.getSqlMgrTrans( - // { id: base.project_id }, - // ncMeta, - // base - // ); - - const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: base.project_id }); - - switch (column.uidt) { - case UITypes.Lookup: - case UITypes.Rollup: - case UITypes.QrCode: - case UITypes.Barcode: - case UITypes.Formula: - await Column.delete(req.params.columnId); - break; - case UITypes.LinkToAnotherRecord: - { - const relationColOpt = - await column.getColOptions(); - const childColumn = await relationColOpt.getChildColumn(); - const childTable = await childColumn.getModel(); - - const parentColumn = await relationColOpt.getParentColumn(); - const parentTable = await parentColumn.getModel(); - - switch (relationColOpt.type) { - case 'bt': - case 'hm': - { - await deleteHmOrBtRelation({ - relationColOpt, - base, - childColumn, - childTable, - parentColumn, - parentTable, - sqlMgr, - // ncMeta - }); - } - break; - case 'mm': - { - const mmTable = await relationColOpt.getMMModel(); - const mmParentCol = await relationColOpt.getMMParentColumn(); - const mmChildCol = await relationColOpt.getMMChildColumn(); - - await deleteHmOrBtRelation( - { - relationColOpt: null, - parentColumn: parentColumn, - childTable: mmTable, - sqlMgr, - parentTable: parentTable, - childColumn: mmParentCol, - base, - // ncMeta - }, - true - ); - - await deleteHmOrBtRelation( - { - relationColOpt: null, - parentColumn: childColumn, - childTable: mmTable, - sqlMgr, - parentTable: childTable, - childColumn: mmChildCol, - base, - // ncMeta - }, - true - ); - const columnsInRelatedTable: Column[] = await relationColOpt - .getRelatedTable() - .then((m) => m.getColumns()); - - for (const c of columnsInRelatedTable) { - if (c.uidt !== UITypes.LinkToAnotherRecord) continue; - const colOpt = - await c.getColOptions(); - if ( - colOpt.type === 'mm' && - colOpt.fk_parent_column_id === childColumn.id && - colOpt.fk_child_column_id === parentColumn.id && - colOpt.fk_mm_model_id === mmTable.id && - colOpt.fk_mm_parent_column_id === mmChildCol.id && - colOpt.fk_mm_child_column_id === mmParentCol.id - ) { - await Column.delete(c.id); - break; - } - } - - await Column.delete(relationColOpt.fk_column_id); - - // delete bt columns in m2m table - await mmTable.getColumns(); - for (const c of mmTable.columns) { - if (c.uidt !== UITypes.LinkToAnotherRecord) continue; - const colOpt = - await c.getColOptions(); - if (colOpt.type === 'bt') { - await Column.delete(c.id); - } - } - - // delete hm columns in parent table - await parentTable.getColumns(); - for (const c of parentTable.columns) { - if (c.uidt !== UITypes.LinkToAnotherRecord) continue; - const colOpt = - await c.getColOptions(); - if (colOpt.fk_related_model_id === mmTable.id) { - await Column.delete(c.id); - } - } - - // delete hm columns in child table - await childTable.getColumns(); - for (const c of childTable.columns) { - if (c.uidt !== UITypes.LinkToAnotherRecord) continue; - const colOpt = - await c.getColOptions(); - if (colOpt.fk_related_model_id === mmTable.id) { - await Column.delete(c.id); - } - } - - // retrieve columns in m2m table again - await mmTable.getColumns(); - - // ignore deleting table if it has more than 2 columns - // the expected 2 columns would be table1_id & table2_id - if (mmTable.columns.length === 2) { - await mmTable.delete(); - } - } - break; - } - } - Tele.emit('evt', { evt_type: 'raltion:deleted' }); - break; - case UITypes.ForeignKey: { - NcError.notImplemented(); - break; - } - // @ts-ignore - case UITypes.SingleSelect: { - if (column.uidt === UITypes.SingleSelect) { - if (await KanbanView.IsColumnBeingUsedAsGroupingField(column.id)) { - NcError.badRequest( - `The column '${column.column_name}' is being used in Kanban View. Please delete Kanban View first.` - ); - } - } - /* falls through to default */ - } - default: { - const tableUpdateBody = { - ...table, - tn: table.table_name, - originalColumns: table.columns.map((c) => ({ - ...c, - cn: c.column_name, - cno: c.column_name, - })), - columns: table.columns.map((c) => { - if (c.id === req.params.columnId) { - return { - ...c, - cn: c.column_name, - cno: c.column_name, - altered: Altered.DELETE_COLUMN, - }; - } else { - (c as any).cn = c.column_name; - } - return c; - }), - }; - - await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); - - await Column.delete(req.params.columnId); - } - } - - await Audit.insert({ - project_id: base.project_id, - op_type: AuditOperationTypes.TABLE_COLUMN, - op_sub_type: AuditOperationSubTypes.DELETED, - user: (req as any)?.user?.email, - description: `deleted column ${column.column_name} with alias ${column.title} from table ${table.table_name}`, - ip: (req as any).clientIp, - }).then(() => {}); - - await table.getColumns(); - - const displayValueColumn = mapDefaultDisplayValue(table.columns); - if (displayValueColumn) { - await Model.updatePrimaryColumn( - displayValueColumn.fk_model_id, - displayValueColumn.id - ); - } - - // await ncMeta.commit(); - // await sql-mgr.commit(); - Tele.emit('evt', { evt_type: 'column:deleted' }); - - res.json(table); - // } catch (e) { - // sql-mgr.rollback(); - // ncMeta.rollback(); - // throw e; - // } + res.json(await columnService.columnDelete({ columnId: req.params.columnId })); } -const deleteHmOrBtRelation = async ( - { - relationColOpt, - base, - childColumn, - childTable, - parentColumn, - parentTable, - sqlMgr, - ncMeta = Noco.ncMeta, - }: { - relationColOpt: LinkToAnotherRecordColumn; - base: Base; - childColumn: Column; - childTable: Model; - parentColumn: Column; - parentTable: Model; - sqlMgr: SqlMgrv2; - ncMeta?: NcMetaIO; - }, - ignoreFkDelete = false -) => { - let foreignKeyName; - - // if relationColOpt is not provided, extract it from child table - // and get the foreign key name for dropping the foreign key - if (!relationColOpt) { - foreignKeyName = ( - ( - await childTable.getColumns().then((cols) => { - return cols?.find((c) => { - return ( - c.uidt === UITypes.LinkToAnotherRecord && - c.colOptions.fk_related_model_id === parentTable.id && - (c.colOptions as LinkToAnotherRecordType).fk_child_column_id === - childColumn.id && - (c.colOptions as LinkToAnotherRecordType).fk_parent_column_id === - parentColumn.id - ); - }); - }) - ).colOptions as LinkToAnotherRecordType - ).fk_index_name; - } else { - foreignKeyName = relationColOpt.fk_index_name; - } - - // todo: handle relation delete exception - try { - await sqlMgr.sqlOpPlus(base, 'relationDelete', { - childColumn: childColumn.column_name, - childTable: childTable.table_name, - parentTable: parentTable.table_name, - parentColumn: parentColumn.column_name, - foreignKeyName, - }); - } catch (e) { - console.log(e); - } - - if (!relationColOpt) return; - const columnsInRelatedTable: Column[] = await relationColOpt - .getRelatedTable() - .then((m) => m.getColumns()); - const relType = relationColOpt.type === 'bt' ? 'hm' : 'bt'; - for (const c of columnsInRelatedTable) { - if (c.uidt !== UITypes.LinkToAnotherRecord) continue; - const colOpt = await c.getColOptions(); - if ( - colOpt.fk_parent_column_id === parentColumn.id && - colOpt.fk_child_column_id === childColumn.id && - colOpt.type === relType - ) { - await Column.delete(c.id, ncMeta); - break; - } - } - - // delete virtual columns - await Column.delete(relationColOpt.fk_column_id, ncMeta); - - if (!ignoreFkDelete) { - const cTable = await Model.getWithInfo({ - id: childTable.id, - }); - const tableUpdateBody = { - ...cTable, - tn: cTable.table_name, - originalColumns: cTable.columns.map((c) => ({ - ...c, - cn: c.column_name, - cno: c.column_name, - })), - columns: cTable.columns.map((c) => { - if (c.id === childColumn.id) { - return { - ...c, - cn: c.column_name, - cno: c.column_name, - altered: Altered.DELETE_COLUMN, - }; - } else { - (c as any).cn = c.column_name; - } - return c; - }), - }; - - await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); - } - // delete foreign key column - await Column.delete(childColumn.id, ncMeta); -}; - -async function createColumnIndex({ - column, - sqlMgr, - base, - indexName = null, - nonUnique = true, -}: { - column: Column; - sqlMgr: SqlMgrv2; - base: Base; - indexName?: string; - nonUnique?: boolean; -}) { - const model = await column.getModel(); - const indexArgs = { - columns: [column.column_name], - tn: model.table_name, - non_unique: nonUnique, - indexName, - }; - sqlMgr.sqlOpPlus(base, 'indexCreate', indexArgs); -} const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/filterController.ts b/packages/nocodb/src/lib/controllers/filterController.ts index 45b18b0104..6ca705b88b 100644 --- a/packages/nocodb/src/lib/controllers/filterController.ts +++ b/packages/nocodb/src/lib/controllers/filterController.ts @@ -16,15 +16,8 @@ import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import { getAjvValidatorMw } from '../meta/api/helpers'; // @ts-ignore -export async function filterGet(req: Request, res: Response, next) { - try { - const filter = await Filter.get(req.params.filterId); - - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } +export async function filterGet(req: Request, res: Response) { + res.json(await Filter.get(req.params.filterId)); } // @ts-ignore @@ -33,14 +26,11 @@ export async function filterList( res: Response, next ) { - try { - const filter = await Filter.rootFilterList({ viewId: req.params.viewId }); - - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } + res.json( + await Filter.rootFilterList({ + viewId: req.params.viewId, + }) + ); } // @ts-ignore export async function filterChildrenRead( diff --git a/packages/nocodb/src/lib/meta/api/helpers/columnHelpers.ts b/packages/nocodb/src/lib/meta/api/helpers/columnHelpers.ts index daf9344126..6c73c3dec9 100644 --- a/packages/nocodb/src/lib/meta/api/helpers/columnHelpers.ts +++ b/packages/nocodb/src/lib/meta/api/helpers/columnHelpers.ts @@ -1,9 +1,9 @@ import { customAlphabet } from 'nanoid'; import { + BoolType, ColumnReqType, LinkToAnotherRecordType, LookupColumnReqType, - BoolType, RelationTypes, RollupColumnReqType, TableType, @@ -77,9 +77,7 @@ export async function createHmAndBtColumn( } } -export async function validateRollupPayload( - payload: ColumnReqType & { uidt: UITypes } -) { +export async function validateRollupPayload(payload: ColumnReqType | Column) { validateParams( [ 'title', @@ -125,7 +123,7 @@ export async function validateRollupPayload( } export async function validateLookupPayload( - payload: ColumnReqType & { uidt: UITypes }, + payload: ColumnReqType, columnId?: string ) { validateParams( diff --git a/packages/nocodb/src/lib/services/columnService.ts b/packages/nocodb/src/lib/services/columnService.ts new file mode 100644 index 0000000000..c8c1f80f4e --- /dev/null +++ b/packages/nocodb/src/lib/services/columnService.ts @@ -0,0 +1,1778 @@ +import { + AuditOperationSubTypes, + AuditOperationTypes, + ColumnReqType, + isVirtualCol, + LinkToAnotherColumnReqType, + LinkToAnotherRecordType, + RelationTypes, + substituteColumnAliasWithIdInFormula, substituteColumnIdWithAliasInFormula, + UITypes, +} from 'nocodb-sdk' +import formulaQueryBuilderv2 from '../db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; +import SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2'; +import { Altered } from '../meta/api/columnApis'; +import { + createHmAndBtColumn, + generateFkName, + randomID, + validateLookupPayload, validateRequiredField, + validateRollupPayload, +} from '../meta/api/helpers' +import { NcError } from '../meta/helpers/catchError'; +import getColumnPropsFromUIDT from '../meta/helpers/getColumnPropsFromUIDT'; +import { + getUniqueColumnAliasName, + getUniqueColumnName, +} from '../meta/helpers/getUniqueName'; +import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; +import validateParams from '../meta/helpers/validateParams'; +import NcMetaIO from '../meta/NcMetaIO'; +import Audit from '../models/Audit'; +import Base from '../models/Base'; +import Column from '../models/Column'; +import FormulaColumn from '../models/FormulaColumn' +import KanbanView from '../models/KanbanView'; +import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; +import Model from '../models/Model'; +import Project from '../models/Project'; +import Noco from '../Noco'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; + +import { Tele } from 'nc-help'; +import { MetaTable } from '../utils/globals' + +export async function columnUpdate(param: { + columnId: string; + column: ColumnReqType & {colOptions?: any}, + cookie?: any, +}) { + const column = await Column.get({ colId: param.columnId }); + + const table = await Model.getWithInfo({ + id: column.fk_model_id, + }); + + const base = await Base.get(table.base_id); + + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + + const sqlClientType = sqlClient.knex.clientType(); + + const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType); + + if (param.column.column_name.length > mxColumnLength) { + NcError.badRequest( + `Column name ${param.column.column_name} exceeds ${mxColumnLength} characters` + ); + } + + if ( + !isVirtualCol(param.column) && + !(await Column.checkTitleAvailable({ + column_name: param.column.column_name, + fk_model_id: column.fk_model_id, + exclude_id: param.columnId, + })) + ) { + NcError.badRequest('Duplicate column name'); + } + if ( + !(await Column.checkAliasAvailable({ + title: param.column.title, + fk_model_id: column.fk_model_id, + exclude_id: param.columnId, + })) + ) { + NcError.badRequest('Duplicate column alias'); + } + + let colBody = { ...param.column } as Column & { formula?: string; formula_raw?: string }; + if ( + [ + UITypes.Lookup, + UITypes.Rollup, + UITypes.LinkToAnotherRecord, + UITypes.Formula, + UITypes.QrCode, + UITypes.Barcode, + UITypes.ForeignKey, + ].includes(column.uidt) + ) { + if (column.uidt === colBody.uidt) { + if ([UITypes.QrCode, UITypes.Barcode].includes(column.uidt)) { + await Column.update(column.id, { + ...column, + ...colBody, + } as Column); + } else if (column.uidt === UITypes.Formula) { + colBody.formula = await substituteColumnAliasWithIdInFormula( + colBody.formula_raw || colBody.formula, + table.columns + ); + + try { + // test the query to see if it is valid in db level + const dbDriver = NcConnectionMgrv2.get(base); + await formulaQueryBuilderv2(colBody.formula, null, dbDriver, table); + } catch (e) { + console.error(e); + NcError.badRequest('Invalid Formula'); + } + + await Column.update(column.id, { + // title: colBody.title, + ...column, + ...colBody, + }); + } else if (colBody.title !== column.title) { + await Column.updateAlias(param.columnId, { + title: colBody.title, + }); + } + await updateRollupOrLookup(colBody, column); + } else { + NcError.notImplemented( + `Updating ${colBody.uidt} => ${colBody.uidt} is not implemented` + ); + } + } else if ( + [ + UITypes.Lookup, + UITypes.Rollup, + UITypes.LinkToAnotherRecord, + UITypes.Formula, + UITypes.QrCode, + UITypes.Barcode, + UITypes.ForeignKey, + ].includes(colBody.uidt) + ) { + NcError.notImplemented( + `Updating ${colBody.uidt} => ${colBody.uidt} is not implemented` + ); + } else if ( + [UITypes.SingleSelect, UITypes.MultiSelect].includes(colBody.uidt) + ) { + colBody = getColumnPropsFromUIDT(colBody, base); + + const baseModel = await Model.getBaseModelSQL({ + id: table.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + if (colBody.colOptions?.options) { + const supportedDrivers = ['mysql', 'mysql2', 'pg', 'mssql', 'sqlite3']; + const dbDriver = NcConnectionMgrv2.get(base); + const driverType = dbDriver.clientType(); + + // MultiSelect to SingleSelect + if ( + column.uidt === UITypes.MultiSelect && + colBody.uidt === UITypes.SingleSelect + ) { + if (driverType === 'mysql' || driverType === 'mysql2') { + await dbDriver.raw( + `UPDATE ?? SET ?? = SUBSTRING_INDEX(??, ',', 1) WHERE ?? LIKE '%,%';`, + [ + table.table_name, + column.column_name, + column.column_name, + column.column_name, + ] + ); + } else if (driverType === 'pg') { + await dbDriver.raw(`UPDATE ?? SET ?? = split_part(??, ',', 1);`, [ + table.table_name, + column.column_name, + column.column_name, + ]); + } else if (driverType === 'mssql') { + await dbDriver.raw( + `UPDATE ?? SET ?? = LEFT(cast(?? as varchar(max)), CHARINDEX(',', ??) - 1) WHERE CHARINDEX(',', ??) > 0;`, + [ + table.table_name, + column.column_name, + column.column_name, + column.column_name, + column.column_name, + ] + ); + } else if (driverType === 'sqlite3') { + await dbDriver.raw( + `UPDATE ?? SET ?? = substr(??, 1, instr(??, ',') - 1) WHERE ?? LIKE '%,%';`, + [ + table.table_name, + column.column_name, + column.column_name, + column.column_name, + column.column_name, + ] + ); + } + } + + // Handle migrations + if (column.colOptions?.options) { + for (const op of column.colOptions.options.filter( + (el) => el.order === null + )) { + op.title = op.title.replace(/^'/, '').replace(/'$/, ''); + } + } + + // Handle default values + const optionTitles = colBody.colOptions.options.map((el) => + el.title.replace(/'/g, "''") + ); + if (colBody.cdf) { + if (colBody.uidt === UITypes.SingleSelect) { + if (!optionTitles.includes(colBody.cdf.replace(/'/g, "''"))) { + NcError.badRequest( + `Default value '${colBody.cdf}' is not a select option.` + ); + } + } else { + for (const cdf of colBody.cdf.split(',')) { + if (!optionTitles.includes(cdf.replace(/'/g, "''"))) { + NcError.badRequest( + `Default value '${cdf}' is not a select option.` + ); + } + } + } + + // handle single quote for default value + if (driverType === 'mysql' || driverType === 'mysql2') { + colBody.cdf = colBody.cdf.replace(/'/g, "'"); + } else { + colBody.cdf = colBody.cdf.replace(/'/g, "''"); + } + + if (driverType === 'pg') { + colBody.cdf = `'${colBody.cdf}'`; + } + } + + // Restrict duplicates + const titles = colBody.colOptions.options.map((el) => el.title); + if ( + titles.some(function (item) { + return titles.indexOf(item) !== titles.lastIndexOf(item); + }) + ) { + NcError.badRequest('Duplicates are not allowed!'); + } + + // Restrict empty options + if ( + titles.some(function (item) { + return item === ''; + }) + ) { + NcError.badRequest('Empty options are not allowed!'); + } + + // Trim end of enum/set + if (colBody.dt === 'enum' || colBody.dt === 'set') { + for (const opt of colBody.colOptions.options) { + opt.title = opt.title.trimEnd(); + } + } + + if (colBody.uidt === UITypes.SingleSelect) { + colBody.dtxp = colBody.colOptions?.options.length + ? `${colBody.colOptions.options + .map((o) => `'${o.title.replace(/'/gi, "''")}'`) + .join(',')}` + : ''; + } else if (colBody.uidt === UITypes.MultiSelect) { + colBody.dtxp = colBody.colOptions?.options.length + ? `${colBody.colOptions.options + .map((o) => { + if (o.title.includes(',')) { + NcError.badRequest("Illegal char(',') for MultiSelect"); + } + return `'${o.title.replace(/'/gi, "''")}'`; + }) + .join(',')}` + : ''; + } + + // Handle empty enum/set for mysql (we restrict empty user options beforehand) + if (driverType === 'mysql' || driverType === 'mysql2') { + if ( + !colBody.colOptions.options.length && + (!colBody.dtxp || colBody.dtxp === '') + ) { + colBody.dtxp = "''"; + } + + if (colBody.dt === 'set') { + if (colBody.colOptions?.options.length > 64) { + colBody.dt = 'text'; + } + } + } + + // Handle option delete + if (column.colOptions?.options) { + for (const option of column.colOptions.options.filter((oldOp) => + colBody.colOptions.options.find((newOp) => newOp.id === oldOp.id) + ? false + : true + )) { + if ( + !supportedDrivers.includes(driverType) && + column.uidt === UITypes.MultiSelect + ) { + NcError.badRequest( + 'Your database not yet supported for this operation. Please remove option from records manually before dropping.' + ); + } + if (column.uidt === UITypes.SingleSelect) { + if (driverType === 'mssql') { + await dbDriver.raw(`UPDATE ?? SET ?? = NULL WHERE ?? LIKE ?`, [ + table.table_name, + column.column_name, + column.column_name, + option.title, + ]); + } else { + await baseModel.bulkUpdateAll( + { where: `(${column.title},eq,${option.title})` }, + { [column.column_name]: null }, + { cookie } + ); + } + } else if (column.uidt === UITypes.MultiSelect) { + if (driverType === 'mysql' || driverType === 'mysql2') { + if (colBody.dt === 'set') { + await dbDriver.raw( + `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), ',')) WHERE FIND_IN_SET(?, ??)`, + [ + table.table_name, + column.column_name, + column.column_name, + option.title, + option.title, + column.column_name, + ] + ); + } else { + await dbDriver.raw( + `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), ','))`, + [ + table.table_name, + column.column_name, + column.column_name, + option.title, + ] + ); + } + } else if (driverType === 'pg') { + await dbDriver.raw( + `UPDATE ?? SET ?? = array_to_string(array_remove(string_to_array(??, ','), ?), ',')`, + [ + table.table_name, + column.column_name, + column.column_name, + option.title, + ] + ); + } else if (driverType === 'mssql') { + await dbDriver.raw( + `UPDATE ?? SET ?? = substring(replace(concat(',', ??, ','), concat(',', ?, ','), ','), 2, len(replace(concat(',', ??, ','), concat(',', ?, ','), ',')) - 2)`, + [ + table.table_name, + column.column_name, + column.column_name, + option.title, + column.column_name, + option.title, + ] + ); + } else if (driverType === 'sqlite3') { + await dbDriver.raw( + `UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ','), ',')`, + [ + table.table_name, + column.column_name, + column.column_name, + option.title, + ] + ); + } + } + } + } + + const interchange = []; + + // Handle option update + if (column.colOptions?.options) { + const old_titles = column.colOptions.options.map((el) => el.title); + for (const option of column.colOptions.options.filter((oldOp) => + colBody.colOptions.options.find( + (newOp) => newOp.id === oldOp.id && newOp.title !== oldOp.title + ) + )) { + if ( + !supportedDrivers.includes(driverType) && + column.uidt === UITypes.MultiSelect + ) { + NcError.badRequest( + 'Your database not yet supported for this operation. Please remove option from records manually before updating.' + ); + } + + const newOp = { + ...colBody.colOptions.options.find((el) => option.id === el.id), + }; + if (old_titles.includes(newOp.title)) { + const def_option = { ...newOp }; + let title_counter = 1; + while (old_titles.includes(newOp.title)) { + newOp.title = `${def_option.title}_${title_counter++}`; + } + interchange.push({ + def_option, + temp_title: newOp.title, + }); + } + + // Append new option before editing + if ( + (driverType === 'mysql' || driverType === 'mysql2') && + (column.dt === 'enum' || column.dt === 'set') + ) { + column.colOptions.options.push({ title: newOp.title }); + + let temp_dtxp = ''; + if (column.uidt === UITypes.SingleSelect) { + temp_dtxp = column.colOptions.options.length + ? `${column.colOptions.options + .map((o) => `'${o.title.replace(/'/gi, "''")}'`) + .join(',')}` + : ''; + } else if (column.uidt === UITypes.MultiSelect) { + temp_dtxp = column.colOptions.options.length + ? `${column.colOptions.options + .map((o) => { + if (o.title.includes(',')) { + NcError.badRequest("Illegal char(',') for MultiSelect"); + throw new Error(''); + } + return `'${o.title.replace(/'/gi, "''")}'`; + }) + .join(',')}` + : ''; + } + + const tableUpdateBody = { + ...table, + tn: table.table_name, + originalColumns: table.columns.map((c) => ({ + ...c, + cn: c.column_name, + cno: c.column_name, + })), + columns: await Promise.all( + table.columns.map(async (c) => { + if (c.id === param.columnId) { + const res = { + ...c, + ...column, + cn: column.column_name, + cno: c.column_name, + dtxp: temp_dtxp, + altered: Altered.UPDATE_COLUMN, + }; + return Promise.resolve(res); + } else { + (c as any).cn = c.column_name; + } + return Promise.resolve(c); + }) + ), + }; + + const sqlMgr = await ProjectMgrv2.getSqlMgr({ + id: base.project_id, + }); + await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); + + await Column.update(param.columnId, { + ...column, + }); + } + + if (column.uidt === UITypes.SingleSelect) { + if (driverType === 'mssql') { + await dbDriver.raw(`UPDATE ?? SET ?? = ? WHERE ?? LIKE ?`, [ + table.table_name, + column.column_name, + newOp.title, + column.column_name, + option.title, + ]); + } else { + await baseModel.bulkUpdateAll( + { where: `(${column.title},eq,${option.title})` }, + { [column.column_name]: newOp.title }, + { cookie } + ); + } + } else if (column.uidt === UITypes.MultiSelect) { + if (driverType === 'mysql' || driverType === 'mysql2') { + if (colBody.dt === 'set') { + await dbDriver.raw( + `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), CONCAT(',', ?, ','))) WHERE FIND_IN_SET(?, ??)`, + [ + table.table_name, + column.column_name, + column.column_name, + option.title, + newOp.title, + option.title, + column.column_name, + ] + ); + } else { + await dbDriver.raw( + `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), CONCAT(',', ?, ',')))`, + [ + table.table_name, + column.column_name, + column.column_name, + option.title, + newOp.title, + ] + ); + } + } else if (driverType === 'pg') { + await dbDriver.raw( + `UPDATE ?? SET ?? = array_to_string(array_replace(string_to_array(??, ','), ?, ?), ',')`, + [ + table.table_name, + column.column_name, + column.column_name, + option.title, + newOp.title, + ] + ); + } else if (driverType === 'mssql') { + await dbDriver.raw( + `UPDATE ?? SET ?? = substring(replace(concat(',', ??, ','), concat(',', ?, ','), concat(',', ?, ',')), 2, len(replace(concat(',', ??, ','), concat(',', ?, ','), concat(',', ?, ','))) - 2)`, + [ + table.table_name, + column.column_name, + column.column_name, + option.title, + newOp.title, + column.column_name, + option.title, + newOp.title, + ] + ); + } else if (driverType === 'sqlite3') { + await dbDriver.raw( + `UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ',' || ? || ','), ',')`, + [ + table.table_name, + column.column_name, + column.column_name, + option.title, + newOp.title, + ] + ); + } + } + } + } + + for (const ch of interchange) { + const newOp = ch.def_option; + if (column.uidt === UITypes.SingleSelect) { + if (driverType === 'mssql') { + await dbDriver.raw(`UPDATE ?? SET ?? = ? WHERE ?? LIKE ?`, [ + table.table_name, + column.column_name, + newOp.title, + column.column_name, + ch.temp_title, + ]); + } else { + await baseModel.bulkUpdateAll( + { where: `(${column.title},eq,${ch.temp_title})` }, + { [column.column_name]: newOp.title }, + { cookie: req } + ); + } + } else if (column.uidt === UITypes.MultiSelect) { + if (driverType === 'mysql' || driverType === 'mysql2') { + if (colBody.dt === 'set') { + await dbDriver.raw( + `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), CONCAT(',', ?, ','))) WHERE FIND_IN_SET(?, ??)`, + [ + table.table_name, + column.column_name, + column.column_name, + ch.temp_title, + newOp.title, + ch.temp_title, + column.column_name, + ] + ); + } else { + await dbDriver.raw( + `UPDATE ?? SET ?? = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ??, ','), CONCAT(',', ?, ','), CONCAT(',', ?, ',')))`, + [ + table.table_name, + column.column_name, + column.column_name, + ch.temp_title, + newOp.title, + ch.temp_title, + column.column_name, + ] + ); + } + } else if (driverType === 'pg') { + await dbDriver.raw( + `UPDATE ?? SET ?? = array_to_string(array_replace(string_to_array(??, ','), ?, ?), ',')`, + [ + table.table_name, + column.column_name, + column.column_name, + ch.temp_title, + newOp.title, + ] + ); + } else if (driverType === 'mssql') { + await dbDriver.raw( + `UPDATE ?? SET ?? = substring(replace(concat(',', ??, ','), concat(',', ?, ','), concat(',', ?, ',')), 2, len(replace(concat(',', ??, ','), concat(',', ?, ','), concat(',', ?, ','))) - 2)`, + [ + table.table_name, + column.column_name, + column.column_name, + ch.temp_title, + newOp.title, + column.column_name, + ch.temp_title, + newOp.title, + ] + ); + } else if (driverType === 'sqlite3') { + await dbDriver.raw( + `UPDATE ?? SET ?? = TRIM(REPLACE(',' || ?? || ',', ',' || ? || ',', ',' || ? || ','), ',')`, + [ + table.table_name, + column.column_name, + column.column_name, + ch.temp_title, + newOp.title, + ] + ); + } + } + } + } + + const tableUpdateBody = { + ...table, + tn: table.table_name, + originalColumns: table.columns.map((c) => ({ + ...c, + cn: c.column_name, + cno: c.column_name, + })), + columns: await Promise.all( + table.columns.map(async (c) => { + if (c.id === param.columnId) { + const res = { + ...c, + ...colBody, + cn: colBody.column_name, + cno: c.column_name, + altered: Altered.UPDATE_COLUMN, + }; + + // update formula with new column name + if (c.column_name != colBody.column_name) { + const formulas = await Noco.ncMeta + .knex(MetaTable.COL_FORMULA) + .where('formula', 'like', `%${c.id}%`); + if (formulas) { + const new_column = c; + new_column.column_name = colBody.column_name; + new_column.title = colBody.title; + for (const f of formulas) { + // the formula with column IDs only + const formula = f.formula; + // replace column IDs with alias to get the new formula_raw + const new_formula_raw = substituteColumnIdWithAliasInFormula( + formula, + [new_column] + ); + await FormulaColumn.update(c.id, { + formula_raw: new_formula_raw, + }); + } + } + } + return Promise.resolve(res); + } else { + (c as any).cn = c.column_name; + } + return Promise.resolve(c); + }) + ), + }; + + const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: base.project_id }); + await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); + + await Column.update(param.columnId, { + ...colBody, + }); + } else { + colBody = getColumnPropsFromUIDT(colBody, base); + const tableUpdateBody = { + ...table, + tn: table.table_name, + originalColumns: table.columns.map((c) => ({ + ...c, + cn: c.column_name, + cno: c.column_name, + })), + columns: await Promise.all( + table.columns.map(async (c) => { + if (c.id === param.columnId) { + const res = { + ...c, + ...colBody, + cn: colBody.column_name, + cno: c.column_name, + altered: Altered.UPDATE_COLUMN, + }; + + // update formula with new column name + if (c.column_name != colBody.column_name) { + const formulas = await Noco.ncMeta + .knex(MetaTable.COL_FORMULA) + .where('formula', 'like', `%${c.id}%`); + if (formulas) { + const new_column = c; + new_column.column_name = colBody.column_name; + new_column.title = colBody.title; + for (const f of formulas) { + // the formula with column IDs only + const formula = f.formula; + // replace column IDs with alias to get the new formula_raw + const new_formula_raw = substituteColumnIdWithAliasInFormula( + formula, + [new_column] + ); + await FormulaColumn.update(c.id, { + formula_raw: new_formula_raw, + }); + } + } + } + return Promise.resolve(res); + } else { + (c as any).cn = c.column_name; + } + return Promise.resolve(c); + }) + ), + }; + + const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: base.project_id }); + await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); + + await Column.update(param.columnId, { + ...colBody, + }); + } + await Audit.insert({ + project_id: base.project_id, + op_type: AuditOperationTypes.TABLE_COLUMN, + op_sub_type: AuditOperationSubTypes.UPDATED, + // user: (req as any)?.user?.email, + description: `updated column ${column.column_name} with alias ${column.title} from table ${table.table_name}`, + // ip: (req as any).clientIp, + }).then(() => {}); + + await table.getColumns(); + Tele.emit('evt', { evt_type: 'column:updated' }); + + return table +} + +export async function columnGet(param: { columnId: string }) { + return Column.get({ colId: param.columnId }); +} + +export async function columnSetAsPrimary(param: { columnId: string }) { + const column = await Column.get({ colId: param.columnId }); + return Model.updatePrimaryColumn(column.fk_model_id, column.id); +} + +export async function columnAdd(param: { + tableId: string; + column: ColumnReqType; +}) { + const table = await Model.getWithInfo({ + id: param.tableId, + }); + + const base = await Base.get(table.base_id); + + const project = await base.getProject(); + + if (param.column.title || param.column.column_name) { + const dbDriver = NcConnectionMgrv2.get(base); + + const sqlClientType = dbDriver.clientType(); + + const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType); + + if ( + (param.column.title || param.column.column_name).length > mxColumnLength + ) { + NcError.badRequest( + `Column name ${ + param.column.title || param.column.column_name + } exceeds ${mxColumnLength} characters` + ); + } + } + + if ( + !isVirtualCol(param.column) && + !(await Column.checkTitleAvailable({ + column_name: param.column.column_name, + fk_model_id: param.tableId, + })) + ) { + NcError.badRequest('Duplicate column name'); + } + if ( + !(await Column.checkAliasAvailable({ + title: param.column.title || param.column.column_name, + fk_model_id: param.tableId, + })) + ) { + NcError.badRequest('Duplicate column alias'); + } + + let colBody: any = param.column; + switch (colBody.uidt) { + case UITypes.Rollup: + { + await validateRollupPayload(param.column); + + await Column.insert({ + ...colBody, + fk_model_id: table.id, + }); + } + break; + case UITypes.Lookup: + { + await validateLookupPayload(param.column); + + await Column.insert({ + ...colBody, + fk_model_id: table.id, + }); + } + break; + + case UITypes.LinkToAnotherRecord: + await createLTARColumn({ ...param, base, project }); + Tele.emit('evt', { evt_type: 'relation:created' }); + break; + + case UITypes.QrCode: + await Column.insert({ + ...colBody, + fk_model_id: table.id, + }); + break; + case UITypes.Barcode: + await Column.insert({ + ...colBody, + fk_model_id: table.id, + }); + break; + case UITypes.Formula: + colBody.formula = await substituteColumnAliasWithIdInFormula( + colBody.formula_raw || colBody.formula, + table.columns + ); + + try { + // test the query to see if it is valid in db level + const dbDriver = NcConnectionMgrv2.get(base); + await formulaQueryBuilderv2(colBody.formula, null, dbDriver, table); + } catch (e) { + console.error(e); + NcError.badRequest('Invalid Formula'); + } + + await Column.insert({ + ...colBody, + fk_model_id: table.id, + }); + + break; + default: + { + colBody = getColumnPropsFromUIDT(colBody, base); + if (colBody.uidt === UITypes.Duration) { + colBody.dtxp = '20'; + // by default, colBody.dtxs is 2 + // Duration column needs more that that + colBody.dtxs = '4'; + } + + if ( + [UITypes.SingleSelect, UITypes.MultiSelect].includes(colBody.uidt) + ) { + const dbDriver = NcConnectionMgrv2.get(base); + const driverType = dbDriver.clientType(); + const optionTitles = colBody.colOptions.options.map((el) => + el.title.replace(/'/g, "''") + ); + + // this is not used for select columns and cause issue for MySQL + colBody.dtxs = ''; + // Handle default values + if (colBody.cdf) { + if (colBody.uidt === UITypes.SingleSelect) { + if (!optionTitles.includes(colBody.cdf.replace(/'/g, "''"))) { + NcError.badRequest( + `Default value '${colBody.cdf}' is not a select option.` + ); + } + } else { + for (const cdf of colBody.cdf.split(',')) { + if (!optionTitles.includes(cdf.replace(/'/g, "''"))) { + NcError.badRequest( + `Default value '${cdf}' is not a select option.` + ); + } + } + } + + // handle single quote for default value + if (driverType === 'mysql' || driverType === 'mysql2') { + colBody.cdf = colBody.cdf.replace(/'/g, "'"); + } else { + colBody.cdf = colBody.cdf.replace(/'/g, "''"); + } + + if (driverType === 'pg') { + colBody.cdf = `'${colBody.cdf}'`; + } + } + + // Restrict duplicates + const titles = colBody.colOptions.options.map((el) => el.title); + if ( + titles.some(function (item) { + return titles.indexOf(item) !== titles.lastIndexOf(item); + }) + ) { + NcError.badRequest('Duplicates are not allowed!'); + } + + // Restrict empty options + if ( + titles.some(function (item) { + return item === ''; + }) + ) { + NcError.badRequest('Empty options are not allowed!'); + } + + // Trim end of enum/set + if (colBody.dt === 'enum' || colBody.dt === 'set') { + for (const opt of colBody.colOptions.options) { + opt.title = opt.title.trimEnd(); + } + } + + if (colBody.uidt === UITypes.SingleSelect) { + colBody.dtxp = colBody.colOptions?.options.length + ? `${colBody.colOptions.options + .map((o) => `'${o.title.replace(/'/gi, "''")}'`) + .join(',')}` + : ''; + } else if (colBody.uidt === UITypes.MultiSelect) { + colBody.dtxp = colBody.colOptions?.options.length + ? `${colBody.colOptions.options + .map((o) => { + if (o.title.includes(',')) { + NcError.badRequest("Illegal char(',') for MultiSelect"); + } + return `'${o.title.replace(/'/gi, "''")}'`; + }) + .join(',')}` + : ''; + } + + // Handle empty enum/set for mysql (we restrict empty user options beforehand) + if (driverType === 'mysql' || driverType === 'mysql2') { + if ( + !colBody.colOptions.options.length && + (!colBody.dtxp || colBody.dtxp === '') + ) { + colBody.dtxp = "''"; + } + + if (colBody.dt === 'set') { + if (colBody.colOptions?.options.length > 64) { + colBody.dt = 'text'; + } + } + } + } + + const tableUpdateBody = { + ...table, + tn: table.table_name, + originalColumns: table.columns.map((c) => ({ + ...c, + cn: c.column_name, + })), + columns: [ + ...table.columns.map((c) => ({ ...c, cn: c.column_name })), + { + ...colBody, + cn: colBody.column_name, + altered: Altered.NEW_COLUMN, + }, + ], + }; + + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: base.project_id }); + await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); + + const columns: Array< + Omit & { + cn: string; + system?: boolean; + } + > = (await sqlClient.columnList({ tn: table.table_name }))?.data?.list; + + const insertedColumnMeta = + columns.find((c) => c.cn === colBody.column_name) || ({} as any); + + await Column.insert({ + ...colBody, + ...insertedColumnMeta, + dtxp: [UITypes.MultiSelect, UITypes.SingleSelect].includes( + colBody.uidt as any + ) + ? colBody.dtxp + : insertedColumnMeta.dtxp, + fk_model_id: table.id, + }); + } + break; + } + + await table.getColumns(); + + await Audit.insert({ + project_id: base.project_id, + op_type: AuditOperationTypes.TABLE_COLUMN, + op_sub_type: AuditOperationSubTypes.CREATED, + // user: (req as any)?.user?.email, + description: `created column ${colBody.column_name} with alias ${colBody.title} from table ${table.table_name}`, + // ip: (req as any).clientIp, + }).then(() => {}); + + Tele.emit('evt', { evt_type: 'column:created' }); + + return table + +} + +export async function columnDelete(param: { columnId: string }) { + const column = await Column.get({ colId: param.columnId }); + const table = await Model.getWithInfo({ + id: column.fk_model_id, + }); + const base = await Base.get(table.base_id); + + // const ncMeta = await Noco.ncMeta.startTransaction(); + // const sql-mgr = await ProjectMgrv2.getSqlMgrTrans( + // { id: base.project_id }, + // ncMeta, + // base + // ); + + const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: base.project_id }); + + switch (column.uidt) { + case UITypes.Lookup: + case UITypes.Rollup: + case UITypes.QrCode: + case UITypes.Barcode: + case UITypes.Formula: + await Column.delete(param.columnId); + break; + case UITypes.LinkToAnotherRecord: + { + const relationColOpt = + await column.getColOptions(); + const childColumn = await relationColOpt.getChildColumn(); + const childTable = await childColumn.getModel(); + + const parentColumn = await relationColOpt.getParentColumn(); + const parentTable = await parentColumn.getModel(); + + switch (relationColOpt.type) { + case 'bt': + case 'hm': + { + await deleteHmOrBtRelation({ + relationColOpt, + base, + childColumn, + childTable, + parentColumn, + parentTable, + sqlMgr, + // ncMeta + }); + } + break; + case 'mm': + { + const mmTable = await relationColOpt.getMMModel(); + const mmParentCol = await relationColOpt.getMMParentColumn(); + const mmChildCol = await relationColOpt.getMMChildColumn(); + + await deleteHmOrBtRelation( + { + relationColOpt: null, + parentColumn: parentColumn, + childTable: mmTable, + sqlMgr, + parentTable: parentTable, + childColumn: mmParentCol, + base, + // ncMeta + }, + true + ); + + await deleteHmOrBtRelation( + { + relationColOpt: null, + parentColumn: childColumn, + childTable: mmTable, + sqlMgr, + parentTable: childTable, + childColumn: mmChildCol, + base, + // ncMeta + }, + true + ); + const columnsInRelatedTable: Column[] = await relationColOpt + .getRelatedTable() + .then((m) => m.getColumns()); + + for (const c of columnsInRelatedTable) { + if (c.uidt !== UITypes.LinkToAnotherRecord) continue; + const colOpt = + await c.getColOptions(); + if ( + colOpt.type === 'mm' && + colOpt.fk_parent_column_id === childColumn.id && + colOpt.fk_child_column_id === parentColumn.id && + colOpt.fk_mm_model_id === mmTable.id && + colOpt.fk_mm_parent_column_id === mmChildCol.id && + colOpt.fk_mm_child_column_id === mmParentCol.id + ) { + await Column.delete(c.id); + break; + } + } + + await Column.delete(relationColOpt.fk_column_id); + + // delete bt columns in m2m table + await mmTable.getColumns(); + for (const c of mmTable.columns) { + if (c.uidt !== UITypes.LinkToAnotherRecord) continue; + const colOpt = + await c.getColOptions(); + if (colOpt.type === 'bt') { + await Column.delete(c.id); + } + } + + // delete hm columns in parent table + await parentTable.getColumns(); + for (const c of parentTable.columns) { + if (c.uidt !== UITypes.LinkToAnotherRecord) continue; + const colOpt = + await c.getColOptions(); + if (colOpt.fk_related_model_id === mmTable.id) { + await Column.delete(c.id); + } + } + + // delete hm columns in child table + await childTable.getColumns(); + for (const c of childTable.columns) { + if (c.uidt !== UITypes.LinkToAnotherRecord) continue; + const colOpt = + await c.getColOptions(); + if (colOpt.fk_related_model_id === mmTable.id) { + await Column.delete(c.id); + } + } + + // retrieve columns in m2m table again + await mmTable.getColumns(); + + // ignore deleting table if it has more than 2 columns + // the expected 2 columns would be table1_id & table2_id + if (mmTable.columns.length === 2) { + await mmTable.delete(); + } + } + break; + } + } + Tele.emit('evt', { evt_type: 'raltion:deleted' }); + break; + case UITypes.ForeignKey: { + NcError.notImplemented(); + break; + } + // @ts-ignore + case UITypes.SingleSelect: { + if (column.uidt === UITypes.SingleSelect) { + if (await KanbanView.IsColumnBeingUsedAsGroupingField(column.id)) { + NcError.badRequest( + `The column '${column.column_name}' is being used in Kanban View. Please delete Kanban View first.` + ); + } + } + /* falls through to default */ + } + default: { + const tableUpdateBody = { + ...table, + tn: table.table_name, + originalColumns: table.columns.map((c) => ({ + ...c, + cn: c.column_name, + cno: c.column_name, + })), + columns: table.columns.map((c) => { + if (c.id === param.columnId) { + return { + ...c, + cn: c.column_name, + cno: c.column_name, + altered: Altered.DELETE_COLUMN, + }; + } else { + (c as any).cn = c.column_name; + } + return c; + }), + }; + + await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); + + await Column.delete(param.columnId); + } + } + + await Audit.insert({ + project_id: base.project_id, + op_type: AuditOperationTypes.TABLE_COLUMN, + op_sub_type: AuditOperationSubTypes.DELETED, + // user: (req as any)?.user?.email, + description: `deleted column ${column.column_name} with alias ${column.title} from table ${table.table_name}`, + // ip: (req as any).clientIp, + }).then(() => {}); + + await table.getColumns(); + + const displayValueColumn = mapDefaultDisplayValue(table.columns); + if (displayValueColumn) { + await Model.updatePrimaryColumn( + displayValueColumn.fk_model_id, + displayValueColumn.id + ); + } + + Tele.emit('evt', { evt_type: 'column:deleted' }); + + return table; +} + +const deleteHmOrBtRelation = async ( + { + relationColOpt, + base, + childColumn, + childTable, + parentColumn, + parentTable, + sqlMgr, + ncMeta = Noco.ncMeta, + }: { + relationColOpt: LinkToAnotherRecordColumn; + base: Base; + childColumn: Column; + childTable: Model; + parentColumn: Column; + parentTable: Model; + sqlMgr: SqlMgrv2; + ncMeta?: NcMetaIO; + }, + ignoreFkDelete = false +) => { + let foreignKeyName; + + // if relationColOpt is not provided, extract it from child table + // and get the foreign key name for dropping the foreign key + if (!relationColOpt) { + foreignKeyName = ( + ( + await childTable.getColumns().then((cols) => { + return cols?.find((c) => { + return ( + c.uidt === UITypes.LinkToAnotherRecord && + c.colOptions.fk_related_model_id === parentTable.id && + (c.colOptions as LinkToAnotherRecordType).fk_child_column_id === + childColumn.id && + (c.colOptions as LinkToAnotherRecordType).fk_parent_column_id === + parentColumn.id + ); + }); + }) + ).colOptions as LinkToAnotherRecordType + ).fk_index_name; + } else { + foreignKeyName = relationColOpt.fk_index_name; + } + + // todo: handle relation delete exception + try { + await sqlMgr.sqlOpPlus(base, 'relationDelete', { + childColumn: childColumn.column_name, + childTable: childTable.table_name, + parentTable: parentTable.table_name, + parentColumn: parentColumn.column_name, + foreignKeyName, + }); + } catch (e) { + console.log(e); + } + + if (!relationColOpt) return; + const columnsInRelatedTable: Column[] = await relationColOpt + .getRelatedTable() + .then((m) => m.getColumns()); + const relType = relationColOpt.type === 'bt' ? 'hm' : 'bt'; + for (const c of columnsInRelatedTable) { + if (c.uidt !== UITypes.LinkToAnotherRecord) continue; + const colOpt = await c.getColOptions(); + if ( + colOpt.fk_parent_column_id === parentColumn.id && + colOpt.fk_child_column_id === childColumn.id && + colOpt.type === relType + ) { + await Column.delete(c.id, ncMeta); + break; + } + } + + // delete virtual columns + await Column.delete(relationColOpt.fk_column_id, ncMeta); + + if (!ignoreFkDelete) { + const cTable = await Model.getWithInfo({ + id: childTable.id, + }); + const tableUpdateBody = { + ...cTable, + tn: cTable.table_name, + originalColumns: cTable.columns.map((c) => ({ + ...c, + cn: c.column_name, + cno: c.column_name, + })), + columns: cTable.columns.map((c) => { + if (c.id === childColumn.id) { + return { + ...c, + cn: c.column_name, + cno: c.column_name, + altered: Altered.DELETE_COLUMN, + }; + } else { + (c as any).cn = c.column_name; + } + return c; + }), + }; + + await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); + } + // delete foreign key column + await Column.delete(childColumn.id, ncMeta); +}; + +async function createLTARColumn(param: { + tableId: string; + column: ColumnReqType; + base: Base; + project: Project; +}) { + validateParams(['parentId', 'childId', 'type'], param.column); + + // get parent and child models + const parent = await Model.getWithInfo({ + id: (param.column as LinkToAnotherColumnReqType).parentId, + }); + const child = await Model.getWithInfo({ + id: (param.column as LinkToAnotherColumnReqType).childId, + }); + let childColumn: Column; + + const sqlMgr = await ProjectMgrv2.getSqlMgr({ + id: param.base.project_id, + }); + if ( + (param.column as LinkToAnotherColumnReqType).type === 'hm' || + (param.column as LinkToAnotherColumnReqType).type === 'bt' + ) { + // populate fk column name + const fkColName = getUniqueColumnName( + await child.getColumns(), + `${parent.table_name}_id` + ); + + let foreignKeyName; + { + // create foreign key + const newColumn = { + cn: fkColName, + + title: fkColName, + column_name: fkColName, + rqd: false, + pk: false, + ai: false, + cdf: null, + dt: parent.primaryKey.dt, + dtxp: parent.primaryKey.dtxp, + dtxs: parent.primaryKey.dtxs, + un: parent.primaryKey.un, + altered: Altered.NEW_COLUMN, + }; + const tableUpdateBody = { + ...child, + tn: child.table_name, + originalColumns: child.columns.map((c) => ({ + ...c, + cn: c.column_name, + })), + columns: [ + ...child.columns.map((c) => ({ + ...c, + cn: c.column_name, + })), + newColumn, + ], + }; + + await sqlMgr.sqlOpPlus(param.base, 'tableUpdate', tableUpdateBody); + + const { id } = await Column.insert({ + ...newColumn, + uidt: UITypes.ForeignKey, + fk_model_id: child.id, + }); + + childColumn = await Column.get({ colId: id }); + + // ignore relation creation if virtual + if (!(param.column as LinkToAnotherColumnReqType).virtual) { + foreignKeyName = generateFkName(parent, child); + // create relation + await sqlMgr.sqlOpPlus(param.base, 'relationCreate', { + childColumn: fkColName, + childTable: child.table_name, + parentTable: parent.table_name, + onDelete: 'NO ACTION', + onUpdate: 'NO ACTION', + type: 'real', + parentColumn: parent.primaryKey.column_name, + foreignKeyName, + }); + } + + // todo: create index for virtual relations as well + // create index for foreign key in pg + if (param.base.type === 'pg') { + await createColumnIndex({ + column: new Column({ + ...newColumn, + fk_model_id: child.id, + }), + base: param.base, + sqlMgr, + }); + } + } + await createHmAndBtColumn( + child, + parent, + childColumn, + (param.column as LinkToAnotherColumnReqType).type as RelationTypes, + (param.column as LinkToAnotherColumnReqType).title, + foreignKeyName, + (param.column as LinkToAnotherColumnReqType).virtual + ); + } else if ((param.column as LinkToAnotherColumnReqType).type === 'mm') { + const aTn = `${param.project?.prefix ?? ''}_nc_m2m_${randomID()}`; + const aTnAlias = aTn; + + const parentPK = parent.primaryKey; + const childPK = child.primaryKey; + + const associateTableCols = []; + + const parentCn = 'table1_id'; + const childCn = 'table2_id'; + + associateTableCols.push( + { + cn: childCn, + column_name: childCn, + title: childCn, + rqd: true, + pk: true, + ai: false, + cdf: null, + dt: childPK.dt, + dtxp: childPK.dtxp, + dtxs: childPK.dtxs, + un: childPK.un, + altered: 1, + uidt: UITypes.ForeignKey, + }, + { + cn: parentCn, + column_name: parentCn, + title: parentCn, + rqd: true, + pk: true, + ai: false, + cdf: null, + dt: parentPK.dt, + dtxp: parentPK.dtxp, + dtxs: parentPK.dtxs, + un: parentPK.un, + altered: 1, + uidt: UITypes.ForeignKey, + } + ); + + await sqlMgr.sqlOpPlus(param.base, 'tableCreate', { + tn: aTn, + _tn: aTnAlias, + columns: associateTableCols, + }); + + const assocModel = await Model.insert(param.project.id, param.base.id, { + table_name: aTn, + title: aTnAlias, + // todo: sanitize + mm: true, + columns: associateTableCols, + }); + + let foreignKeyName1; + let foreignKeyName2; + + if (!(param.column as LinkToAnotherColumnReqType).virtual) { + foreignKeyName1 = generateFkName(parent, child); + foreignKeyName2 = generateFkName(parent, child); + + const rel1Args = { + ...param.column, + childTable: aTn, + childColumn: parentCn, + parentTable: parent.table_name, + parentColumn: parentPK.column_name, + type: 'real', + foreignKeyName: foreignKeyName1, + }; + const rel2Args = { + ...param.column, + childTable: aTn, + childColumn: childCn, + parentTable: child.table_name, + parentColumn: childPK.column_name, + type: 'real', + foreignKeyName: foreignKeyName2, + }; + + await sqlMgr.sqlOpPlus(param.base, 'relationCreate', rel1Args); + await sqlMgr.sqlOpPlus(param.base, 'relationCreate', rel2Args); + } + const parentCol = (await assocModel.getColumns())?.find( + (c) => c.column_name === parentCn + ); + const childCol = (await assocModel.getColumns())?.find( + (c) => c.column_name === childCn + ); + + await createHmAndBtColumn( + assocModel, + child, + childCol, + null, + null, + foreignKeyName1, + (param.column as LinkToAnotherColumnReqType).virtual, + true + ); + await createHmAndBtColumn( + assocModel, + parent, + parentCol, + null, + null, + foreignKeyName2, + (param.column as LinkToAnotherColumnReqType).virtual, + true + ); + + await Column.insert({ + title: getUniqueColumnAliasName( + await child.getColumns(), + `${parent.title} List` + ), + uidt: UITypes.LinkToAnotherRecord, + type: 'mm', + + // ref_db_alias + fk_model_id: child.id, + // db_type: + + fk_child_column_id: childPK.id, + fk_parent_column_id: parentPK.id, + + fk_mm_model_id: assocModel.id, + fk_mm_child_column_id: childCol.id, + fk_mm_parent_column_id: parentCol.id, + fk_related_model_id: parent.id, + }); + await Column.insert({ + title: getUniqueColumnAliasName( + await parent.getColumns(), + param.column.title ?? `${child.title} List` + ), + + uidt: UITypes.LinkToAnotherRecord, + type: 'mm', + + fk_model_id: parent.id, + + fk_child_column_id: parentPK.id, + fk_parent_column_id: childPK.id, + + fk_mm_model_id: assocModel.id, + fk_mm_child_column_id: parentCol.id, + fk_mm_parent_column_id: childCol.id, + fk_related_model_id: child.id, + }); + + // todo: create index for virtual relations as well + // create index for foreign key in pg + if (param.base.type === 'pg') { + await createColumnIndex({ + column: new Column({ + ...associateTableCols[0], + fk_model_id: assocModel.id, + }), + base: param.base, + sqlMgr, + }); + await createColumnIndex({ + column: new Column({ + ...associateTableCols[1], + fk_model_id: assocModel.id, + }), + base: param.base, + sqlMgr, + }); + } + } +} + +export async function createColumnIndex({ + column, + sqlMgr, + base, + indexName = null, + nonUnique = true, +}: { + column: Column; + sqlMgr: SqlMgrv2; + base: Base; + indexName?: string; + nonUnique?: boolean; +}) { + const model = await column.getModel(); + const indexArgs = { + columns: [column.column_name], + tn: model.table_name, + non_unique: nonUnique, + indexName, + }; + sqlMgr.sqlOpPlus(base, 'indexCreate', indexArgs); +} + +async function updateRollupOrLookup(colBody: any, column: Column) { + if ( + UITypes.Lookup === column.uidt && + validateRequiredField(colBody, [ + 'fk_lookup_column_id', + 'fk_relation_column_id', + ]) + ) { + await validateLookupPayload(colBody, column.id); + await Column.update(column.id, colBody); + } else if ( + UITypes.Rollup === column.uidt && + validateRequiredField(colBody, [ + 'fk_relation_column_id', + 'fk_rollup_column_id', + 'rollup_function', + ]) + ) { + await validateRollupPayload(colBody); + await Column.update(column.id, colBody); + } +} diff --git a/packages/nocodb/src/lib/services/filterService.ts b/packages/nocodb/src/lib/services/filterService.ts new file mode 100644 index 0000000000..dd7e8799af --- /dev/null +++ b/packages/nocodb/src/lib/services/filterService.ts @@ -0,0 +1,11 @@ +import Filter from '../models/Filter' + +export async function filterGet(param: { filterId: string }) { + const filter = await Filter.get(param.filterId) + return filter +} + +export async function filterList(param: { viewId: string }) { + const filter = await Filter.rootFilterList({ viewId: param.viewId }) + return filter +} diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index afe5d99b06..45cc1853f4 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -1,2 +1,4 @@ // export * as projectService from './projectService'; export * as tableService from './tableService'; +export * as columnService from './columnService'; +export * as filterService from './filterService'; diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts index 2bf2c4ca9f..a3459860fc 100644 --- a/packages/nocodb/src/lib/services/tableService.ts +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -32,7 +32,7 @@ export function reorderTable(param: { tableId: string; order: any }) { return Model.updateOrder(param.tableId, param.order); } -export async function deleteTable(param: { tableId: string; user: User }) { +export async function tableDelete(param: { tableId: string; user: User }) { const table = await Model.getByIdOrName({ id: param.tableId }); await table.getColumns(); @@ -193,7 +193,7 @@ export async function getAccessibleTables(param: { : (tableList.filter((t) => !t.mm) as Model[]); } -export async function createTable(args: { +export async function tableCreate(args: { projectId: string; baseId?: string; table: TableReqType; From 89adb8dafad8bd366dfa2256beeed72e14cc499e Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 28 Feb 2023 17:21:53 +0530 Subject: [PATCH 05/64] refcator: handler to service-controller - project, column, view, hook, base (WIP) Signed-off-by: Pranav C --- .../src/lib/controllers/filterController.ts | 130 ++++------- .../src/lib/controllers/projectController.ts | 177 +++------------ .../src/lib/controllers/sortController.ts | 50 +++-- .../src/lib/controllers/viewController.ts | 83 +++---- packages/nocodb/src/lib/models/Hook.ts | 8 +- packages/nocodb/src/lib/models/Project.ts | 2 +- packages/nocodb/src/lib/models/View.ts | 3 +- packages/nocodb/src/lib/models/index.ts | 36 +++ .../src/lib/services/apiTokenService.ts | 30 +++ .../nocodb/src/lib/services/baseService.ts | 78 +++++++ .../nocodb/src/lib/services/filterService.ts | 57 +++++ .../nocodb/src/lib/services/hookService.ts | 123 ++++++++++ packages/nocodb/src/lib/services/index.ts | 6 +- .../nocodb/src/lib/services/projectService.ts | 211 ++++++++++++++---- .../nocodb/src/lib/services/sortService.ts | 33 +++ .../nocodb/src/lib/services/viewService.ts | 85 +++++++ 16 files changed, 757 insertions(+), 355 deletions(-) create mode 100644 packages/nocodb/src/lib/models/index.ts create mode 100644 packages/nocodb/src/lib/services/apiTokenService.ts create mode 100644 packages/nocodb/src/lib/services/baseService.ts create mode 100644 packages/nocodb/src/lib/services/hookService.ts create mode 100644 packages/nocodb/src/lib/services/sortService.ts create mode 100644 packages/nocodb/src/lib/services/viewService.ts diff --git a/packages/nocodb/src/lib/controllers/filterController.ts b/packages/nocodb/src/lib/controllers/filterController.ts index 6ca705b88b..d83f3e22b7 100644 --- a/packages/nocodb/src/lib/controllers/filterController.ts +++ b/packages/nocodb/src/lib/controllers/filterController.ts @@ -1,119 +1,73 @@ import { Request, Response, Router } from 'express'; -// @ts-ignore -import Model from '../models/Model'; -import { Tele } from 'nc-help'; -// @ts-ignore -import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -// @ts-ignore -import { Table, TableList, TableListParams, TableReq } from 'nocodb-sdk'; -// @ts-ignore -import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; -// @ts-ignore -import Project from '../models/Project'; -import Filter from '../models/Filter'; +import { FilterReqType } from 'nocodb-sdk'; +import { getAjvValidatorMw } from '../meta/api/helpers'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; + +import { filterService } from '../services'; // @ts-ignore export async function filterGet(req: Request, res: Response) { - res.json(await Filter.get(req.params.filterId)); + res.json(await filterService.filterGet({ filterId: req.params.filterId })); } // @ts-ignore -export async function filterList( - req: Request, - res: Response, - next -) { +export async function filterList(req: Request, res: Response) { res.json( - await Filter.rootFilterList({ + await filterService.filterList({ viewId: req.params.viewId, }) ); } + // @ts-ignore -export async function filterChildrenRead( - req: Request, - res: Response, - next -) { - try { - const filter = await Filter.parentFilterList({ - parentId: req.params.filterParentId, - }); +export async function filterChildrenRead(req: Request, res: Response) { + const filter = await filterService.filterChildrenList({ + filterId: req.params.filterParentId, + }); - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } + res.json(filter); } -export async function filterCreate( - req: Request, - res, - next -) { - try { - const filter = await Filter.insert({ - ...req.body, - fk_view_id: req.params.viewId, - }); +export async function filterCreate(req: Request, res) { + const filter = await filterService.filterCreate({ + filter: req.body, + viewId: req.params.viewId, + }); + res.json(filter); +} - Tele.emit('evt', { evt_type: 'filter:created' }); - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } +export async function filterUpdate(req, res) { + const filter = await filterService.filterUpdate({ + filterId: req.params.filterId, + filter: req.body, + }); + res.json(filter); } -// @ts-ignore -export async function filterUpdate(req, res, next) { - try { - const filter = await Filter.update(req.params.filterId, { - ...req.body, - fk_view_id: req.params.viewId, - }); - Tele.emit('evt', { evt_type: 'filter:updated' }); - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } +export async function filterDelete(req: Request, res: Response) { + const filter = await filterService.filterDelete({ + filterId: req.params.filterId, + }); + res.json(filter); } -// @ts-ignore -export async function filterDelete(req: Request, res: Response, next) { - try { - const filter = await Filter.delete(req.params.filterId); - Tele.emit('evt', { evt_type: 'filter:deleted' }); - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } +export async function hookFilterList(req: Request, res: Response) { + res.json( + await filterService.hookFilterList({ + hookId: req.params.hookId, + }) + ); } -export async function hookFilterList( - req: Request, - res: Response +export async function hookFilterCreate( + req: Request, + res ) { - const filter = await Filter.rootFilterListByHook({ + const filter = await filterService.hookFilterCreate({ + filter: req.body, hookId: req.params.hookId, }); - - res.json(filter); -} - -export async function hookFilterCreate(req: Request, res) { - const filter = await Filter.insert({ - ...req.body, - fk_hook_id: req.params.hookId, - }); - - Tele.emit('evt', { evt_type: 'hookFilter:created' }); res.json(filter); } diff --git a/packages/nocodb/src/lib/controllers/projectController.ts b/packages/nocodb/src/lib/controllers/projectController.ts index 89351269b8..5e65b8efdd 100644 --- a/packages/nocodb/src/lib/controllers/projectController.ts +++ b/packages/nocodb/src/lib/controllers/projectController.ts @@ -1,27 +1,20 @@ import { Request, Response } from 'express'; -import { OrgUserRoles, ProjectType } from 'nocodb-sdk'; +import { ProjectType } from 'nocodb-sdk'; import Project from '../models/Project'; import { ProjectListType } from 'nocodb-sdk'; -import DOMPurify from 'isomorphic-dompurify'; import { packageVersion } from '../utils/packageVersion'; import { Tele } from 'nc-help'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import syncMigration from '../meta/helpers/syncMigration'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import ProjectUser from '../models/ProjectUser'; -import { customAlphabet } from 'nanoid'; import Noco from '../Noco'; import isDocker from 'is-docker'; -import { NcError } from '../meta/helpers/catchError'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { extractPropsAndSanitize } from '../meta/helpers/extractProps'; -import NcConfigFactory from '../utils/NcConfigFactory'; -import { promisify } from 'util'; -import { getAjvValidatorMw, populateMeta } from '../meta/api/helpers'; +import { getAjvValidatorMw } from '../meta/api/helpers'; import Filter from '../models/Filter'; -const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 4); +import { projectService } from '../services'; // // Project CRUD @@ -29,13 +22,12 @@ export async function projectGet( req: Request, res: Response ) { - const project = await Project.getWithInfo(req.params.projectId); - - // delete datasource connection details - project.bases?.forEach((b) => { - ['config'].forEach((k) => delete b[k]); + const project = await projectService.getProjectWithInfo({ + projectId: req.params.projectId, }); + projectService.sanitizeProject(project); + res.json(project); } @@ -43,154 +35,49 @@ export async function projectUpdate( req: Request, res: Response ) { - const project = await Project.getWithInfo(req.params.projectId); - - const data: Partial = extractPropsAndSanitize(req?.body, [ - 'title', - 'meta', - 'color', - ]); - - if ( - data?.title && - project.title !== data.title && - (await Project.getByTitle(data.title)) - ) { - NcError.badRequest('Project title already in use'); - } + const project = await projectService.projectUpdate({ + projectId: req.params.projectId, + project: req.body, + }); - const result = await Project.update(req.params.projectId, data); - Tele.emit('evt', { evt_type: 'project:update' }); - res.json(result); + res.json(project); } export async function projectList( req: Request & { user: { id: string; roles: string } }, - res: Response, - next + res: Response ) { - try { - const projects = req.user?.roles?.includes(OrgUserRoles.SUPER_ADMIN) - ? await Project.list(req.query) - : await ProjectUser.getProjectsList(req.user.id, req.query); + const projects = await projectService.projectList({ + user: req.user, + query: req.query, + }); - res // todo: pagination - .json( - new PagedResponseImpl(projects as ProjectType[], { - count: projects.length, - limit: projects.length, - }) - ); - } catch (e) { - console.log(e); - next(e); - } + res.json( + new PagedResponseImpl(projects as ProjectType[], { + count: projects.length, + limit: projects.length, + }) + ); } export async function projectDelete( req: Request, - res: Response + res: Response ) { - const result = await Project.softDelete(req.params.projectId); - Tele.emit('evt', { evt_type: 'project:deleted' }); - res.json(result); -} - -// -// - -async function projectCreate(req: Request, res) { - const projectBody = req.body; - if (!projectBody.external) { - const ranId = nanoid(); - projectBody.prefix = `nc_${ranId}__`; - projectBody.is_meta = true; - if (process.env.NC_MINIMAL_DBS) { - // if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each project - // each file will be named as nc_.db - const fs = require('fs'); - const toolDir = NcConfigFactory.getToolDir(); - const nanoidv2 = customAlphabet( - '1234567890abcdefghijklmnopqrstuvwxyz', - 14 - ); - if (!(await promisify(fs.exists)(`${toolDir}/nc_minimal_dbs`))) { - await promisify(fs.mkdir)(`${toolDir}/nc_minimal_dbs`); - } - const dbId = nanoidv2(); - const projectTitle = DOMPurify.sanitize(projectBody.title); - projectBody.prefix = ''; - projectBody.bases = [ - { - type: 'sqlite3', - config: { - client: 'sqlite3', - connection: { - client: 'sqlite3', - database: projectTitle, - connection: { - filename: `${toolDir}/nc_minimal_dbs/${projectTitle}_${dbId}.db`, - }, - }, - }, - inflection_column: 'camelize', - inflection_table: 'camelize', - }, - ]; - } else { - const db = Noco.getConfig().meta?.db; - projectBody.bases = [ - { - type: db?.client, - config: null, - is_meta: true, - inflection_column: 'camelize', - inflection_table: 'camelize', - }, - ]; - } - } else { - if (process.env.NC_CONNECT_TO_EXTERNAL_DB_DISABLED) { - NcError.badRequest('Connecting to external db is disabled'); - } - projectBody.is_meta = false; - } - - if (projectBody?.title.length > 50) { - NcError.badRequest('Project title exceeds 50 characters'); - } - - if (await Project.getByTitle(projectBody?.title)) { - NcError.badRequest('Project title already in use'); - } - - projectBody.title = DOMPurify.sanitize(projectBody.title); - projectBody.slug = projectBody.title; - - const project = await Project.createProject(projectBody); - await ProjectUser.insert({ - fk_user_id: (req as any).user.id, - project_id: project.id, - roles: 'owner', + const deleted = await projectService.projectSoftDelete({ + projectId: req.params.projectId, }); - await syncMigration(project); - - // populate metadata if existing table - for (const base of await project.getBases()) { - const info = await populateMeta(base, project); + res.json(deleted); +} - Tele.emit('evt_api_created', info); - delete base.config; - } - Tele.emit('evt', { - evt_type: 'project:created', - xcdb: !projectBody.external, +async function projectCreate(req: Request, res) { + const project = await projectService.projectCreate({ + project: req.body, + user: req['user'], }); - Tele.emit('evt', { evt_type: 'project:rest' }); - res.json(project); } diff --git a/packages/nocodb/src/lib/controllers/sortController.ts b/packages/nocodb/src/lib/controllers/sortController.ts index c1ee743b51..c9ee69515a 100644 --- a/packages/nocodb/src/lib/controllers/sortController.ts +++ b/packages/nocodb/src/lib/controllers/sortController.ts @@ -1,52 +1,52 @@ import { Request, Response, Router } from 'express'; -// @ts-ignore -import Model from '../models/Model'; -import { Tele } from 'nc-help'; -// @ts-ignore +import { getAjvValidatorMw } from '../meta/api/helpers' import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import { SortListType, SortType, TableType } from 'nocodb-sdk'; -// @ts-ignore -import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; -// @ts-ignore -import Project from '../models/Project'; -import Sort from '../models/Sort'; +import { SortListType, SortReqType } from 'nocodb-sdk'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; -// @ts-ignore -export async function sortGet(req: Request, res: Response) {} +import { sortService } from '../services'; // @ts-ignore export async function sortList( req: Request, res: Response ) { - const sortList = await Sort.list({ viewId: req.params.viewId }); + const sortList = await sortService.sortList({ + viewId: req.params.viewId, + }); res.json({ sorts: new PagedResponseImpl(sortList), }); } // @ts-ignore -export async function sortCreate(req: Request, res) { - const sort = await Sort.insert({ - ...req.body, - fk_view_id: req.params.viewId, - } as Sort); - Tele.emit('evt', { evt_type: 'sort:created' }); +export async function sortCreate(req: Request, res) { + const sort = await sortService.sortCreate({ + sort: req.body, + viewId: req.params.viewId, + }); res.json(sort); } export async function sortUpdate(req, res) { - const sort = await Sort.update(req.params.sortId, req.body); - Tele.emit('evt', { evt_type: 'sort:updated' }); + const sort = await sortService.sortUpdate({ + sortId: req.params.sortId, + sort: req.body, + }); res.json(sort); } export async function sortDelete(req: Request, res: Response) { - Tele.emit('evt', { evt_type: 'sort:deleted' }); - const sort = await Sort.delete(req.params.sortId); + const sort = await sortService.sortDelete({ + sortId: req.params.sortId, + }); + res.json(sort); +} +export async function sortGet(req: Request, res: Response) { + const sort = await sortService.sortGet({ + sortId: req.params.sortId, + }); res.json(sort); } @@ -62,11 +62,13 @@ router.post( getAjvValidatorMw('swagger.json#/components/schemas/SortReq'), ncMetaAclMw(sortCreate, 'sortCreate') ); + router.get( '/api/v1/db/meta/sorts/:sortId', metaApiMetrics, ncMetaAclMw(sortGet, 'sortGet') ); + router.patch( '/api/v1/db/meta/sorts/:sortId', metaApiMetrics, diff --git a/packages/nocodb/src/lib/controllers/viewController.ts b/packages/nocodb/src/lib/controllers/viewController.ts index 4cc86a2d54..e0a1cc6631 100644 --- a/packages/nocodb/src/lib/controllers/viewController.ts +++ b/packages/nocodb/src/lib/controllers/viewController.ts @@ -1,43 +1,21 @@ import { Request, Response, Router } from 'express'; -// @ts-ignore -import Model from '../models/Model'; -import { Tele } from 'nc-help'; -// @ts-ignore import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -// @ts-ignore -import { Table, TableReq, ViewList } from 'nocodb-sdk'; -// @ts-ignore -import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; -// @ts-ignore -import Project from '../models/Project'; -import View from '../models/View'; +import { View } from '../models'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { xcVisibilityMetaGet } from './modelVisibilityController'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { viewService } from '../services'; + // @ts-ignore -export async function viewGet(req: Request, res: Response
) {} +export async function viewGet(req: Request, res: Response) {} // @ts-ignore export async function viewList( req: Request, - res: Response + res: Response ) { - const model = await Model.get(req.params.tableId); - - const viewList = await xcVisibilityMetaGet( - // req.params.projectId, - // req.params.baseId, - model.project_id, - [model] - ); - - //await View.list(req.params.tableId) - const filteredViewList = viewList.filter((view: any) => { - return Object.keys((req as any).session?.passport?.user?.roles).some( - (role) => - (req as any)?.session?.passport?.user?.roles[role] && - !view.disabled[role] - ); + const filteredViewList = await viewService.viewList({ + tableId: req.params.tableId, + user: (req as any).session?.passport?.user, }); res.json(new PagedResponseImpl(filteredViewList)); @@ -48,8 +26,7 @@ export async function shareView( req: Request, res: Response ) { - Tele.emit('evt', { evt_type: 'sharedView:generated-link' }); - res.json(await View.share(req.params.viewId)); + res.json(await viewService.shareView({ viewId: req.params.viewId })); } // @ts-ignore @@ -57,48 +34,56 @@ export async function viewCreate(req: Request, res, next) {} // @ts-ignore export async function viewUpdate(req, res) { - const result = await View.update(req.params.viewId, req.body); - Tele.emit('evt', { evt_type: 'vtable:updated', show_as: result.type }); + const result = await viewService.viewUpdate({ + viewId: req.params.viewId, + view: req.body, + }); res.json(result); } // @ts-ignore export async function viewDelete(req: Request, res: Response, next) { - const result = await View.delete(req.params.viewId); - Tele.emit('evt', { evt_type: 'vtable:deleted' }); + const result = await viewService.viewDelete({ viewId: req.params.viewId }); res.json(result); } async function shareViewUpdate(req: Request, res) { - Tele.emit('evt', { evt_type: 'sharedView:updated' }); - res.json(await View.update(req.params.viewId, req.body)); + res.json( + await viewService.shareViewUpdate({ + viewId: req.params.viewId, + sharedView: req.body, + }) + ); } async function shareViewDelete(req: Request, res) { - Tele.emit('evt', { evt_type: 'sharedView:deleted' }); - res.json(await View.sharedViewDelete(req.params.viewId)); + res.json(await viewService.shareViewDelete({ viewId: req.params.viewId })); } async function showAllColumns(req: Request, res) { res.json( - await View.showAllColumns( - req.params.viewId, - (req.query?.ignoreIds || []) - ) + await viewService.showAllColumns({ + viewId: req.params.viewId, + ignoreIds: (req.query?.ignoreIds || []), + }) ); } async function hideAllColumns(req: Request, res) { res.json( - await View.hideAllColumns( - req.params.viewId, - (req.query?.ignoreIds || []) - ) + await viewService.hideAllColumns({ + viewId: req.params.viewId, + ignoreIds: (req.query?.ignoreIds || []), + }) ); } async function shareViewList(req: Request, res) { - res.json(await View.shareViewList(req.params.tableId)); + res.json( + await viewService.shareViewList({ + tableId: req.params.tableId, + }) + ); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/models/Hook.ts b/packages/nocodb/src/lib/models/Hook.ts index 7c32d09bbd..47a640d8d9 100644 --- a/packages/nocodb/src/lib/models/Hook.ts +++ b/packages/nocodb/src/lib/models/Hook.ts @@ -1,4 +1,4 @@ -import { HookType } from 'nocodb-sdk'; +import { BoolType, HookType } from 'nocodb-sdk' import { CacheDelDirection, CacheGetType, @@ -21,16 +21,16 @@ export default class Hook implements HookType { type?: string; event?: 'after' | 'before'; operation?: 'insert' | 'delete' | 'update'; - async?: boolean; + async?: BoolType; payload?: string; url?: string; headers?: string; - condition?: boolean; + condition?: BoolType; notification?: string; retries?: number; retry_interval?: number; timeout?: number; - active?: boolean; + active?: BoolType; project_id?: string; base_id?: string; diff --git a/packages/nocodb/src/lib/models/Project.ts b/packages/nocodb/src/lib/models/Project.ts index 5ee799d1ab..a3774b96d9 100644 --- a/packages/nocodb/src/lib/models/Project.ts +++ b/packages/nocodb/src/lib/models/Project.ts @@ -1,6 +1,6 @@ import Base from './/Base'; import Noco from '../Noco'; -import { BoolType, MetaType, ProjectType } from 'nocodb-sdk'; +import { BoolType, MetaType, ProjectType } from 'nocodb-sdk' import { CacheDelDirection, CacheGetType, diff --git a/packages/nocodb/src/lib/models/View.ts b/packages/nocodb/src/lib/models/View.ts index 41e17d6d67..e453da37ea 100644 --- a/packages/nocodb/src/lib/models/View.ts +++ b/packages/nocodb/src/lib/models/View.ts @@ -15,6 +15,7 @@ import GridViewColumn from './GridViewColumn'; import Sort from './Sort'; import Filter from './Filter'; import { + BoolType, ColumnReqType, isSystemColumn, UITypes, @@ -942,7 +943,7 @@ export default class View implements ViewType { body: { title?: string; order?: number; - show_system_fields?: boolean; + show_system_fields?: BoolType; lock_type?: string; password?: string; uuid?: string; diff --git a/packages/nocodb/src/lib/models/index.ts b/packages/nocodb/src/lib/models/index.ts new file mode 100644 index 0000000000..534c8ce025 --- /dev/null +++ b/packages/nocodb/src/lib/models/index.ts @@ -0,0 +1,36 @@ +export { default as ApiToken } from './ApiToken'; +export { default as Audit } from './Audit'; +export { default as BarcodeColumn } from './BarcodeColumn'; +export { default as Base } from './Base'; +export { default as Column } from './Column'; +export { default as Filter } from './Filter'; +export { default as FormulaColumn } from './FormulaColumn'; +export { default as FormView } from './FormView'; +export { default as FormViewColumn } from './FormViewColumn'; +export { default as GalleryView } from './GalleryView'; +export { default as GalleryViewColumn } from './GalleryViewColumn'; +export { default as GridView } from './GridView'; +export { default as GridViewColumn } from './GridViewColumn'; +export { default as Hook } from './Hook'; +export { default as HookFilter } from './HookFilter'; +export { default as HookLog } from './HookLog'; +export { default as KanbanView } from './KanbanView'; +export { default as KanbanViewColumn } from './KanbanViewColumn'; +export { default as LinkToAnotherRecordColumn } from './LinkToAnotherRecordColumn'; +export { default as LookupColumn } from './LookupColumn'; +export { default as MapView } from './MapView'; +export { default as MapViewColumn } from './MapViewColumn'; +export { default as Model } from './Model'; +export { default as ModelRoleVisibility } from './ModelRoleVisibility'; +export { default as Plugin } from './Plugin'; +export { default as Project } from './Project'; +export { default as ProjectUser } from './ProjectUser'; +export { default as QrCodeColumn } from './QrCodeColumn'; +export { default as RollupColumn } from './RollupColumn'; +export { default as SelectOption } from './SelectOption'; +export { default as Sort } from './Sort'; +export { default as Store } from './Store'; +export { default as SyncLogs } from './SyncLogs'; +export { default as SyncSource } from './SyncSource'; +export { default as User } from './User'; +export { default as View } from './View'; diff --git a/packages/nocodb/src/lib/services/apiTokenService.ts b/packages/nocodb/src/lib/services/apiTokenService.ts new file mode 100644 index 0000000000..aa106328f6 --- /dev/null +++ b/packages/nocodb/src/lib/services/apiTokenService.ts @@ -0,0 +1,30 @@ +import { ApiTokenReqType, OrgUserRoles } from 'nocodb-sdk'; +import { Tele } from 'nc-help'; +import { NcError } from '../meta/helpers/catchError'; +import ApiToken from '../models/ApiToken'; +import User from '../models/User'; + +export async function apiTokenList(param: { userId: string }) { + return ApiToken.list(param.userId); +} +export async function apiTokenCreate(param: { + userId: string; + tokenBody: ApiTokenReqType; +}) { + Tele.emit('evt', { evt_type: 'apiToken:created' }); + return ApiToken.insert({ ...param.tokenBody, fk_user_id: param.userId }); +} + +export async function apiTokenDelete(param: { token; user: User }) { + const apiToken = await ApiToken.getByToken(param.token); + if ( + !param.user.roles.includes(OrgUserRoles.SUPER_ADMIN) && + apiToken.fk_user_id !== param.user.id + ) { + NcError.notFound('Token not found'); + } + Tele.emit('evt', { evt_type: 'apiToken:deleted' }); + + // todo: verify token belongs to the user + return await ApiToken.delete(param.token); +} diff --git a/packages/nocodb/src/lib/services/baseService.ts b/packages/nocodb/src/lib/services/baseService.ts new file mode 100644 index 0000000000..1f390198c0 --- /dev/null +++ b/packages/nocodb/src/lib/services/baseService.ts @@ -0,0 +1,78 @@ +import Project from '../models/Project'; +import { BaseReqType } from 'nocodb-sdk'; +import { syncBaseMigration } from '../meta/helpers/syncMigration'; +import Base from '../models/Base'; +import { Tele } from 'nc-help'; +import { populateMeta } from '../meta/api/helpers'; + +export async function baseGetWithConfig(param: { baseId: any }) { + const base = await Base.get(param.baseId); + + base.config = base.getConnectionConfig(); + + return base; +} + +export async function baseUpdate(param: { + baseId: string; + base: BaseReqType; + projectId: string; +}) { + const baseBody = param.base; + const project = await Project.getWithInfo(param.projectId); + const base = await Base.updateBase(param.baseId, { + ...baseBody, + type: baseBody.config?.client, + projectId: project.id, + id: param.baseId, + }); + + delete base.config; + + Tele.emit('evt', { + evt_type: 'base:updated', + }); + + return base; +} + +export async function baseList(param: { projectId: string }) { + const bases = await Base.list({ projectId: param.projectId }); + + return bases; +} + +export async function baseDelete(param: { baseId: string }) { + const base = await Base.get(param.baseId); + await base.delete(); + Tele.emit('evt', { evt_type: 'base:deleted' }); + return true; +} + +export async function baseCreate(param: { + projectId: string; + base: BaseReqType; +}) { + // type | base | projectId + const baseBody = param.base; + const project = await Project.getWithInfo(param.projectId); + const base = await Base.createBase({ + ...baseBody, + type: baseBody.config?.client, + projectId: project.id, + }); + + await syncBaseMigration(project, base); + + const info = await populateMeta(base, project); + + Tele.emit('evt_api_created', info); + + delete base.config; + + Tele.emit('evt', { + evt_type: 'base:created', + }); + + return base; +} diff --git a/packages/nocodb/src/lib/services/filterService.ts b/packages/nocodb/src/lib/services/filterService.ts index dd7e8799af..8b40b4b813 100644 --- a/packages/nocodb/src/lib/services/filterService.ts +++ b/packages/nocodb/src/lib/services/filterService.ts @@ -1,4 +1,61 @@ +import { FilterReqType } from 'nocodb-sdk' import Filter from '../models/Filter' +import { Tele } from 'nc-help'; + +export async function hookFilterCreate(param: { filter: FilterReqType; hookId: any }) { + const filter = await Filter.insert({ + ...param.filter, + fk_hook_id: param.hookId, + }) + + Tele.emit('evt', { evt_type: 'hookFilter:created' }) + return filter +} + + +export async function hookFilterList(param: { hookId: any }) { + return Filter.rootFilterListByHook({ hookId: param.hookId }) +} + +export async function filterDelete(param: { filterId: string }) { + await Filter.delete(param.filterId) + Tele.emit('evt', { evt_type: 'filter:deleted' }) + return true; +} + + +export async function filterCreate(param: { + filter: FilterReqType + viewId: string +}) { + const filter = await Filter.insert({ + ...param.filter, + fk_view_id: param.viewId, + }); + + Tele.emit('evt', { evt_type: 'filter:created' }); + + return filter +} +export async function filterUpdate(param: { + filter: FilterReqType + filterId: string +}) { + const filter = await Filter.update(param.filterId, param.filter) + + + Tele.emit('evt', { evt_type: 'filter:updated' }) + + return filter +} + + +export async function filterChildrenList(param: { filterId: any }) { + return Filter.parentFilterList({ + parentId: param.filterId, + }) +} + export async function filterGet(param: { filterId: string }) { const filter = await Filter.get(param.filterId) diff --git a/packages/nocodb/src/lib/services/hookService.ts b/packages/nocodb/src/lib/services/hookService.ts new file mode 100644 index 0000000000..36de0ceb15 --- /dev/null +++ b/packages/nocodb/src/lib/services/hookService.ts @@ -0,0 +1,123 @@ +import { Tele } from 'nc-help'; +import catchError from '../helpers/catchError'; +import { Request, Response, Router } from 'express'; +import { Hook, Model } from '../models'; +import { HookListType, HookReqType, HookTestReqType, HookType, TableReqType } from 'nocodb-sdk' +import { PagedResponseImpl } from '../helpers/PagedResponse'; +import { invokeWebhook } from '../helpers/webhookHelpers'; +import populateSamplePayload from '../helpers/populateSamplePayload'; +import ncMetaAclMw from '../helpers/ncMetaAclMw'; +import { metaApiMetrics } from '../helpers/apiMetrics'; +import { getAjvValidatorMw } from './helpers'; + +export async function hookList( +param: { + tableId: string; +} +) { + // todo: pagination + await Hook.list({ fk_model_id: param.tableId })) +} + +export async function hookCreate( +param: { + tableId: string; + hook: HookReqType +} +) { + Tele.emit('evt', { evt_type: 'webhooks:created' }); + const hook = await Hook.insert({ + ...param.hook, + fk_model_id: param.tableId, + }); + return hook; +} + +export async function hookDelete( +param :{ + hookId: string; +} +) { + Tele.emit('evt', { evt_type: 'webhooks:deleted' }); + await Hook.delete(param.hookId); + return true; +} + +export async function hookUpdate( + param: { + hookId: string; + hook: HookReqType; + } +) { + Tele.emit('evt', { evt_type: 'webhooks:updated' }); + + return await Hook.update(param.hookId, param.hook) +} + +export async function hookTest(param: { + tableId: string; + hookTest: HookTestReqType; +}) { + const model = await Model.getByIdOrName({ id: param.tableId }); + + const { + hook, + payload: { data, user }, + } = param.hookTest; + await invokeWebhook( + new Hook(hook), + model, + data, + user, + (hook as any)?.filters, + true + ); + + Tele.emit('evt', { evt_type: 'webhooks:tested' }); + + return true +} +export async function tableSampleData(param:{ + tableId: string; + operation: 'insert' | 'update'; +}) { + const model = await Model.getByIdOrName({ id: param.tableId }); + + return await populateSamplePayload(model, false, param.operation); +} + +const router = Router({ mergeParams: true }); +router.get( + '/api/v1/db/meta/tables/:tableId/hooks', + metaApiMetrics, + ncMetaAclMw(hookList, 'hookList') +); +router.post( + '/api/v1/db/meta/tables/:tableId/hooks/test', + metaApiMetrics, + getAjvValidatorMw('swagger.json#/components/schemas/HookTestReq'), + ncMetaAclMw(hookTest, 'hookTest') +); +router.post( + '/api/v1/db/meta/tables/:tableId/hooks', + metaApiMetrics, + getAjvValidatorMw('swagger.json#/components/schemas/HookReq'), + ncMetaAclMw(hookCreate, 'hookCreate') +); +router.delete( + '/api/v1/db/meta/hooks/:hookId', + metaApiMetrics, + ncMetaAclMw(hookDelete, 'hookDelete') +); +router.patch( + '/api/v1/db/meta/hooks/:hookId', + metaApiMetrics, + getAjvValidatorMw('swagger.json#/components/schemas/HookReq'), + ncMetaAclMw(hookUpdate, 'hookUpdate') +); +router.get( + '/api/v1/db/meta/tables/:tableId/hooks/samplePayload/:operation', + metaApiMetrics, + catchError(tableSampleData) +); +export default router; diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 45cc1853f4..2d53935889 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -1,4 +1,8 @@ -// export * as projectService from './projectService'; +export * as projectService from './projectService'; export * as tableService from './tableService'; export * as columnService from './columnService'; export * as filterService from './filterService'; +export * as sortService from './sortService'; +export * as baseService from './baseService'; +export * as apiTokenService from './apiTokenService'; +export * as viewService from './viewService'; diff --git a/packages/nocodb/src/lib/services/projectService.ts b/packages/nocodb/src/lib/services/projectService.ts index 1520f6de21..131df14557 100644 --- a/packages/nocodb/src/lib/services/projectService.ts +++ b/packages/nocodb/src/lib/services/projectService.ts @@ -1,42 +1,169 @@ -// // services/project.service.ts -// import { OrgUserRoles, ProjectType } from 'nocodb-sdk' -// import Project from '../models/Project' -// import ProjectUser from '../models/ProjectUser' -// import { customAlphabet } from 'nanoid' -// import { NcError } from './helpers/catchError' -// -// const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 4) -// -// export async function getProjectWithInfo(projectId: string) { -// const project = await Project.getWithInfo(projectId) -// return project -// } -// -// export function sanitizeProject(project: any) { -// const sanitizedProject = { ...project } -// sanitizedProject.bases?.forEach((b: any) => { -// ['config'].forEach((k) => delete b[k]) -// }) -// return sanitizedProject -// } -// -// export async function updateProject(projectId: string, data: Partial) { -// const project = await Project.getWithInfo(projectId) -// if (data?.title && project.title !== data.title && (await Project.getByTitle(data.title))) { -// NcError.badRequest('Project title already in use') -// } -// const result = await Project.update(projectId, data) -// return result -// } -// -// export async function listProjects(user: any) { -// const projects = user?.roles?.includes(OrgUserRoles.SUPER_ADMIN) -// ? await Project.list() -// : await ProjectUser.getProjectsList(user.id) -// return projects as ProjectType[] -// } -// -// export async function softDeleteProject(projectId: string) { -// const result = await Project.softDelete(projectId) -// return result -// } +import DOMPurify from 'isomorphic-dompurify'; +import { OrgUserRoles, ProjectReqType, } from 'nocodb-sdk'; +import { promisify } from 'util'; +import { populateMeta } from '../meta/api/helpers'; +import { extractPropsAndSanitize } from '../meta/helpers/extractProps'; +import syncMigration from '../meta/helpers/syncMigration'; +import Project from '../models/Project'; +import ProjectUser from '../models/ProjectUser'; +import { customAlphabet } from 'nanoid'; +import Noco from '../Noco'; +import NcConfigFactory from '../utils/NcConfigFactory'; +import { NcError } from '../meta/helpers/catchError'; + +export async function projectCreate(param: { + project: ProjectReqType; + user: any; +}) { + const projectBody: ProjectReqType & Record = param.project; + if (!projectBody.external) { + const ranId = nanoid(); + projectBody.prefix = `nc_${ranId}__`; + projectBody.is_meta = true; + if (process.env.NC_MINIMAL_DBS) { + // if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each project + // each file will be named as nc_.db + const fs = require('fs'); + const toolDir = NcConfigFactory.getToolDir(); + const nanoidv2 = customAlphabet( + '1234567890abcdefghijklmnopqrstuvwxyz', + 14 + ); + if (!(await promisify(fs.exists)(`${toolDir}/nc_minimal_dbs`))) { + await promisify(fs.mkdir)(`${toolDir}/nc_minimal_dbs`); + } + const dbId = nanoidv2(); + const projectTitle = DOMPurify.sanitize(projectBody.title); + projectBody.prefix = ''; + projectBody.bases = [ + { + type: 'sqlite3', + config: { + client: 'sqlite3', + connection: { + client: 'sqlite3', + database: projectTitle, + connection: { + filename: `${toolDir}/nc_minimal_dbs/${projectTitle}_${dbId}.db`, + }, + }, + }, + inflection_column: 'camelize', + inflection_table: 'camelize', + }, + ]; + } else { + const db = Noco.getConfig().meta?.db; + projectBody.bases = [ + { + type: db?.client, + config: null, + is_meta: true, + inflection_column: 'camelize', + inflection_table: 'camelize', + }, + ]; + } + } else { + if (process.env.NC_CONNECT_TO_EXTERNAL_DB_DISABLED) { + NcError.badRequest('Connecting to external db is disabled'); + } + projectBody.is_meta = false; + } + + if (projectBody?.title.length > 50) { + NcError.badRequest('Project title exceeds 50 characters'); + } + + if (await Project.getByTitle(projectBody?.title)) { + NcError.badRequest('Project title already in use'); + } + + projectBody.title = DOMPurify.sanitize(projectBody.title); + projectBody.slug = projectBody.title; + + const project = await Project.createProject(projectBody); + await ProjectUser.insert({ + fk_user_id: (param as any).user.id, + project_id: project.id, + roles: 'owner', + }); + + await syncMigration(project); + + // populate metadata if existing table + for (const base of await project.getBases()) { + const info = await populateMeta(base, project); + + Tele.emit('evt_api_created', info); + delete base.config; + } + + Tele.emit('evt', { + evt_type: 'project:created', + xcdb: !projectBody.external, + }); + + Tele.emit('evt', { evt_type: 'project:rest' }); + + project; +} + +const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 4); + +import { Tele } from 'nc-help'; + +export async function getProjectWithInfo(param: { projectId: string }) { + const project = await Project.getWithInfo(param.projectId); + return project; +} + +export async function projectSoftDelete(param: { projectId: any }) { + await Project.softDelete(param.projectId); + Tele.emit('evt', { evt_type: 'project:deleted' }); + return true; +} + +export function sanitizeProject(project: any) { + const sanitizedProject = { ...project }; + sanitizedProject.bases?.forEach((b: any) => { + ['config'].forEach((k) => delete b[k]); + }); + return sanitizedProject; +} + +export async function projectUpdate(param: { + projectId: string; + project: ProjectReqType; +}) { + const project = await Project.getWithInfo(param.projectId); + + const data: Partial = extractPropsAndSanitize( + param?.project as Project, + ['title', 'meta', 'color'] + ); + + if ( + data?.title && + project.title !== data.title && + (await Project.getByTitle(data.title)) + ) { + NcError.badRequest('Project title already in use'); + } + + const result = await Project.update(param.projectId, data); + Tele.emit('evt', { evt_type: 'project:update' }); + + return result; +} + +export async function projectList(param: { + user: { id: string; roles: string }; + query?: any; +}) { + const projects = param.user?.roles?.includes(OrgUserRoles.SUPER_ADMIN) + ? await Project.list(param.query) + : await ProjectUser.getProjectsList(param.user.id, param.query); + + return projects; +} diff --git a/packages/nocodb/src/lib/services/sortService.ts b/packages/nocodb/src/lib/services/sortService.ts new file mode 100644 index 0000000000..4d69e278c8 --- /dev/null +++ b/packages/nocodb/src/lib/services/sortService.ts @@ -0,0 +1,33 @@ +import { SortReqType } from 'nocodb-sdk'; +import Sort from '../models/Sort'; +import { Tele } from 'nc-help'; + +export async function sortGet(param: { sortId: string }) { + return Sort.get(param.sortId); +} + + +export async function sortDelete(param: { sortId: string }) { + await Sort.delete(param.sortId); + Tele.emit('evt', { evt_type: 'sort:deleted' }); + return true; +} + +export async function sortUpdate(param: { sortId: any; sort: SortReqType }) { + const sort = await Sort.update(param.sortId, param.sort); + Tele.emit('evt', { evt_type: 'sort:updated' }); + return sort; +} + +export async function sortCreate(param: { viewId: any; sort: SortReqType }) { + const sort = await Sort.insert({ + ...param.sort, + fk_view_id: param.viewId, + } as Sort); + Tele.emit('evt', { evt_type: 'sort:created' }); + return sort; +} + +export async function sortList(param: { viewId: string }) { + return Sort.list({ viewId: param.viewId }); +} diff --git a/packages/nocodb/src/lib/services/viewService.ts b/packages/nocodb/src/lib/services/viewService.ts new file mode 100644 index 0000000000..4d8cf2f1bc --- /dev/null +++ b/packages/nocodb/src/lib/services/viewService.ts @@ -0,0 +1,85 @@ +import { Model, View } from '../models'; +import { Tele } from 'nc-help'; +import { SharedViewReqType, ViewReqType } from 'nocodb-sdk'; +import { xcVisibilityMetaGet } from '../meta/api/modelVisibilityApis'; + + +export async function viewList(param: { + tableId: string; + user: { + roles: Record; + }; +}) { + const model = await Model.get(param.tableId); + + const viewList = await xcVisibilityMetaGet( + // param.projectId, + // param.baseId, + model.project_id, + [model] + ); + + // todo: user roles + //await View.list(param.tableId) + const filteredViewList = viewList.filter((view: any) => { + return Object.keys(param?.user?.roles).some( + (role) => param?.user?.roles[role] && !view.disabled[role] + ); + }); + + return filteredViewList; +} + +// @ts-ignore +export async function shareView(param: { viewId: string }) { + Tele.emit('evt', { evt_type: 'sharedView:generated-link' }); + return await View.share(param.viewId); +} + +// @ts-ignore +export async function viewUpdate(param: { viewId: string; view: ViewReqType }) { + const result = await View.update(param.viewId, param.view); + Tele.emit('evt', { evt_type: 'vtable:updated', show_as: result.type }); + return result; +} + +// @ts-ignore +export async function viewDelete(param: { viewId: string }) { + await View.delete(param.viewId); + Tele.emit('evt', { evt_type: 'vtable:deleted' }); + return true; +} + +export async function shareViewUpdate(param: { + viewId: string; + sharedView: SharedViewReqType; +}) { + Tele.emit('evt', { evt_type: 'sharedView:updated' }); + return await View.update(param.viewId, param.sharedView); +} + +export async function shareViewDelete(param: { viewId: string }) { + Tele.emit('evt', { evt_type: 'sharedView:deleted' }); + await View.sharedViewDelete(param.viewId); + return true; +} + +export async function showAllColumns(param: { + viewId: string; + ignoreIds?: string[]; +}) { + await View.showAllColumns(param.viewId, param.ignoreIds || []); + return true; +} + +export async function hideAllColumns(param: { + viewId: string; + ignoreIds?: string[]; +}) { + await View.hideAllColumns(param.viewId, param.ignoreIds || []); + return true; +} + +export async function shareViewList(param: { tableId: string }) { + return await View.shareViewList(param.tableId); +} From 10195c84b04aadded8a52bd75811f9f41fc83460 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 28 Feb 2023 19:09:29 +0530 Subject: [PATCH 06/64] refcator: handler to service-controller - form, gallery, grid, kanban, plugin (WIP) Signed-off-by: Pranav C --- .../controllers/formViewColumnController.ts | 11 +- .../lib/controllers/galleryViewController.ts | 29 +- .../src/lib/controllers/hookController.ts | 58 ++- .../src/lib/controllers/pluginController.ts | 21 +- .../nocodb/src/lib/controllers/utilService.ts | 59 +++ packages/nocodb/src/lib/models/FormView.ts | 10 +- packages/nocodb/src/lib/models/GalleryView.ts | 12 +- .../src/lib/services/formViewColumnService.ts | 13 + .../src/lib/services/formViewService.ts | 31 ++ .../src/lib/services/galleryViewService.ts | 31 ++ .../src/lib/services/gridViewService.ts | 26 ++ .../nocodb/src/lib/services/hookService.ts | 84 +--- packages/nocodb/src/lib/services/index.ts | 8 + .../src/lib/services/kanbanViewService.ts | 29 ++ .../nocodb/src/lib/services/pluginService.ts | 30 ++ .../nocodb/src/lib/services/utilService.ts | 381 ++++++++++++++++++ .../nocodb/src/lib/services/viewService.ts | 1 - 17 files changed, 688 insertions(+), 146 deletions(-) create mode 100644 packages/nocodb/src/lib/controllers/utilService.ts create mode 100644 packages/nocodb/src/lib/services/formViewColumnService.ts create mode 100644 packages/nocodb/src/lib/services/formViewService.ts create mode 100644 packages/nocodb/src/lib/services/galleryViewService.ts create mode 100644 packages/nocodb/src/lib/services/gridViewService.ts create mode 100644 packages/nocodb/src/lib/services/kanbanViewService.ts create mode 100644 packages/nocodb/src/lib/services/pluginService.ts create mode 100644 packages/nocodb/src/lib/services/utilService.ts diff --git a/packages/nocodb/src/lib/controllers/formViewColumnController.ts b/packages/nocodb/src/lib/controllers/formViewColumnController.ts index fd1b5ab8e7..532b5eb82a 100644 --- a/packages/nocodb/src/lib/controllers/formViewColumnController.ts +++ b/packages/nocodb/src/lib/controllers/formViewColumnController.ts @@ -1,13 +1,16 @@ import { Request, Response, Router } from 'express'; -import FormViewColumn from '../models/FormViewColumn'; -import { Tele } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import { getAjvValidatorMw } from '../meta/api/helpers'; +import { formViewColumnService } from '../services'; export async function columnUpdate(req: Request, res: Response) { - Tele.emit('evt', { evt_type: 'formViewColumn:updated' }); - res.json(await FormViewColumn.update(req.params.formViewColumnId, req.body)); + res.json( + await formViewColumnService.columnUpdate({ + formViewColumnId: req.params.formViewColumnId, + formViewColumn: req.body, + }) + ); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/galleryViewController.ts b/packages/nocodb/src/lib/controllers/galleryViewController.ts index d682f70fad..947ea6b65a 100644 --- a/packages/nocodb/src/lib/controllers/galleryViewController.ts +++ b/packages/nocodb/src/lib/controllers/galleryViewController.ts @@ -1,29 +1,34 @@ import { Request, Response, Router } from 'express'; -import { GalleryType, ViewTypes } from 'nocodb-sdk'; -import View from '../models/View'; -import GalleryView from '../models/GalleryView'; -import { Tele } from 'nc-help'; +import { GalleryType } from 'nocodb-sdk'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import { getAjvValidatorMw } from '../meta/api/helpers'; +import { galleryViewService } from '../services'; + export async function galleryViewGet(req: Request, res: Response) { - res.json(await GalleryView.get(req.params.galleryViewId)); + res.json( + await galleryViewService.galleryViewGet({ + galleryViewId: req.params.galleryViewId, + }) + ); } export async function galleryViewCreate(req: Request, res) { - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'gallery' }); - const view = await View.insert({ - ...req.body, + const view = await galleryViewService.galleryViewCreate({ + gallery: req.body, // todo: sanitize - fk_model_id: req.params.tableId, - type: ViewTypes.GALLERY, + tableId: req.params.tableId, }); res.json(view); } export async function galleryViewUpdate(req, res) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'gallery' }); - res.json(await GalleryView.update(req.params.galleryViewId, req.body)); + res.json( + await galleryViewService.galleryViewUpdate({ + galleryViewId: req.params.galleryViewId, + gallery: req.body, + }) + ); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/hookController.ts b/packages/nocodb/src/lib/controllers/hookController.ts index 1215b4604a..c17280a9e9 100644 --- a/packages/nocodb/src/lib/controllers/hookController.ts +++ b/packages/nocodb/src/lib/controllers/hookController.ts @@ -1,15 +1,11 @@ -import { Tele } from 'nc-help'; import catchError from '../meta/helpers/catchError'; import { Request, Response, Router } from 'express'; -import Hook from '../models/Hook'; import { HookListType, HookType } from 'nocodb-sdk'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import { invokeWebhook } from '../meta/helpers/webhookHelpers'; -import Model from '../models/Model'; -import populateSamplePayload from '../meta/helpers/populateSamplePayload'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import { getAjvValidatorMw } from '../meta/api/helpers'; +import { hookService } from '../services'; export async function hookList( req: Request, @@ -17,7 +13,9 @@ export async function hookList( ) { // todo: pagination res.json( - new PagedResponseImpl(await Hook.list({ fk_model_id: req.params.tableId })) + new PagedResponseImpl( + await hookService.hookList({ tableId: req.params.tableId }) + ) ); } @@ -25,10 +23,9 @@ export async function hookCreate( req: Request, res: Response ) { - Tele.emit('evt', { evt_type: 'webhooks:created' }); - const hook = await Hook.insert({ - ...req.body, - fk_model_id: req.params.tableId, + const hook = await hookService.hookCreate({ + hook: req.body, + tableId: req.params.tableId, }); res.json(hook); } @@ -37,44 +34,35 @@ export async function hookDelete( req: Request, res: Response ) { - Tele.emit('evt', { evt_type: 'webhooks:deleted' }); - res.json(await Hook.delete(req.params.hookId)); + res.json(await hookService.hookDelete({ hookId: req.params.hookId })); } export async function hookUpdate( req: Request, res: Response ) { - Tele.emit('evt', { evt_type: 'webhooks:updated' }); - - res.json(await Hook.update(req.params.hookId, req.body)); + res.json( + await hookService.hookUpdate({ hookId: req.params.hookId, hook: req.body }) + ); } export async function hookTest(req: Request, res: Response) { - const model = await Model.getByIdOrName({ id: req.params.tableId }); - - const { - hook, - payload: { data, user }, - } = req.body; - await invokeWebhook( - new Hook(hook), - model, - data, - user, - (hook as any)?.filters, - true - ); - - Tele.emit('evt', { evt_type: 'webhooks:tested' }); - + await hookService.hookTest({ + hookTest: req.body, + tableId: req.params.tableId, + }); res.json({ msg: 'Success' }); } -export async function tableSampleData(req: Request, res: Response) { - const model = await Model.getByIdOrName({ id: req.params.tableId }); +export async function tableSampleData(req: Request, res: Response) { res // todo: pagination - .json(await populateSamplePayload(model, false, req.params.operation)); + .json( + await hookService.tableSampleData({ + tableId: req.params.tableId, + // todo: replace any with type + operation: req.params.operation as any, + }) + ); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/pluginController.ts b/packages/nocodb/src/lib/controllers/pluginController.ts index a48ecbf33f..c5bef7e9a0 100644 --- a/packages/nocodb/src/lib/controllers/pluginController.ts +++ b/packages/nocodb/src/lib/controllers/pluginController.ts @@ -1,38 +1,33 @@ import { Request, Response, Router } from 'express'; -import { Tele } from 'nc-help'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import Plugin from '../models/Plugin'; import { PluginType } from 'nocodb-sdk'; -import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import { getAjvValidatorMw } from '../meta/api/helpers'; +import { pluginService } from '../services'; + + export async function pluginList(_req: Request, res: Response) { - res.json(new PagedResponseImpl(await Plugin.list())); + res.json(new PagedResponseImpl(await pluginService.pluginList())); } export async function pluginTest(req: Request, res: Response) { - Tele.emit('evt', { evt_type: 'plugin:tested' }); - res.json(await NcPluginMgrv2.test(req.body)); + res.json(await pluginService.pluginTest({ body: req.body })); } export async function pluginRead(req: Request, res: Response) { - res.json(await Plugin.get(req.params.pluginId)); + res.json(await pluginService.pluginRead({ pluginId: req.params.pluginId })); } export async function pluginUpdate( req: Request, res: Response ) { - const plugin = await Plugin.update(req.params.pluginId, req.body); - Tele.emit('evt', { - evt_type: plugin.active ? 'plugin:installed' : 'plugin:uninstalled', - title: plugin.title, - }); + const plugin = await pluginService.pluginUpdate({ pluginId: req.params.pluginId, plugin:req.body }); res.json(plugin); } export async function isPluginActive(req: Request, res: Response) { - res.json(await Plugin.isPluginActive(req.params.pluginTitle)); + res.json(await pluginService.isPluginActive({ pluginTitle: req.params.pluginTitle })); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/utilService.ts b/packages/nocodb/src/lib/controllers/utilService.ts new file mode 100644 index 0000000000..e5e2b504b6 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/utilService.ts @@ -0,0 +1,59 @@ +// // Project CRUD +import { Request, Response } from 'express'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import catchError from '../meta/helpers/catchError'; +import { utilService } from '../services'; + +export async function testConnection(req: Request, res: Response) { + res.json(await utilService.testConnection({ body: req.body })); +} + +export async function appInfo(req: Request, res: Response) { + res.json( + await utilService.appInfo({ + req: { + ncSiteUrl: (req as any).ncSiteUrl, + }, + }) + ); +} + +export async function versionInfo(_req: Request, res: Response) { + res.json(await utilService.versionInfo()); +} + +export async function appHealth(_: Request, res: Response) { + res.json(await utilService.appHealth()); +} + +export async function axiosRequestMake(req: Request, res: Response) { + res.json(await utilService.axiosRequestMake({ body: req.body })); +} + +export async function aggregatedMetaInfo(_req: Request, res: Response) { + res.json(await utilService.aggregatedMetaInfo()); +} + +export async function urlToDbConfig(req: Request, res: Response) { + res.json( + await utilService.urlToDbConfig({ + body: req.body, + }) + ); +} + +export default (router) => { + router.post( + '/api/v1/db/meta/connection/test', + ncMetaAclMw(testConnection, 'testConnection') + ); + router.get('/api/v1/db/meta/nocodb/info', catchError(appInfo)); + router.post('/api/v1/db/meta/axiosRequestMake', catchError(axiosRequestMake)); + router.get('/api/v1/version', catchError(versionInfo)); + router.get('/api/v1/health', catchError(appHealth)); + router.post('/api/v1/url_to_config', catchError(urlToDbConfig)); + router.get( + '/api/v1/aggregated-meta-info', + ncMetaAclMw(aggregatedMetaInfo, 'aggregatedMetaInfo') + ); +}; diff --git a/packages/nocodb/src/lib/models/FormView.ts b/packages/nocodb/src/lib/models/FormView.ts index ad155f4434..6edbec8746 100644 --- a/packages/nocodb/src/lib/models/FormView.ts +++ b/packages/nocodb/src/lib/models/FormView.ts @@ -1,6 +1,6 @@ import Noco from '../Noco'; import { CacheGetType, CacheScope, MetaTable } from '../utils/globals'; -import { FormType } from 'nocodb-sdk'; +import { BoolType, FormType } from 'nocodb-sdk'; import { deserializeJSON, serializeJSON } from '../utils/serialize'; import FormViewColumn from './FormViewColumn'; import View from './View'; @@ -8,8 +8,8 @@ import NocoCache from '../cache/NocoCache'; import { extractProps } from '../meta/helpers/extractProps'; export default class FormView implements FormType { - show: boolean; - is_default: boolean; + show: BoolType; + is_default: BoolType; order: number; title?: string; heading?: string; @@ -20,8 +20,8 @@ export default class FormView implements FormType { email?: string; banner_image_url?: string; logo_url?: string; - submit_another_form?: boolean; - show_blank_form?: boolean; + submit_another_form?: BoolType; + show_blank_form?: BoolType; fk_view_id: string; columns?: FormViewColumn[]; diff --git a/packages/nocodb/src/lib/models/GalleryView.ts b/packages/nocodb/src/lib/models/GalleryView.ts index 2cf2f6ce8f..141e37cbcc 100644 --- a/packages/nocodb/src/lib/models/GalleryView.ts +++ b/packages/nocodb/src/lib/models/GalleryView.ts @@ -1,24 +1,24 @@ import Noco from '../Noco'; import { CacheGetType, CacheScope, MetaTable } from '../utils/globals'; -import { GalleryColumnType, GalleryType, UITypes } from 'nocodb-sdk'; +import { BoolType, GalleryColumnType, GalleryType, UITypes } from 'nocodb-sdk' import View from './View'; import NocoCache from '../cache/NocoCache'; import { extractProps } from '../meta/helpers/extractProps'; export default class GalleryView implements GalleryType { fk_view_id?: string; - deleted?: boolean; + deleted?: BoolType; order?: number; - next_enabled?: boolean; - prev_enabled?: boolean; + next_enabled?: BoolType; + prev_enabled?: BoolType; cover_image_idx?: number; cover_image?: string; restrict_types?: string; restrict_size?: string; restrict_number?: string; - public?: boolean; + public?: BoolType; password?: string; - show_all_fields?: boolean; + show_all_fields?: BoolType; fk_cover_image_col_id?: string; project_id?: string; diff --git a/packages/nocodb/src/lib/services/formViewColumnService.ts b/packages/nocodb/src/lib/services/formViewColumnService.ts new file mode 100644 index 0000000000..afef0e02f3 --- /dev/null +++ b/packages/nocodb/src/lib/services/formViewColumnService.ts @@ -0,0 +1,13 @@ +import { FormViewColumn } from '../models'; +import { Tele } from 'nc-help'; +export async function columnUpdate(param: { + formViewColumnId: string; + // todo: replace with FormColumnReq + formViewColumn: FormViewColumn; +}) { + Tele.emit('evt', { evt_type: 'formViewColumn:updated' }); + return await FormViewColumn.update( + param.formViewColumnId, + param.formViewColumn + ); +} diff --git a/packages/nocodb/src/lib/services/formViewService.ts b/packages/nocodb/src/lib/services/formViewService.ts new file mode 100644 index 0000000000..d9a4667c72 --- /dev/null +++ b/packages/nocodb/src/lib/services/formViewService.ts @@ -0,0 +1,31 @@ +import { Tele } from 'nc-help'; +import { FormReqType, ViewTypes } from 'nocodb-sdk'; +import { FormView, View } from '../models'; + +export async function formViewGet(param: { formViewId: string }) { + const formViewData = await FormView.getWithInfo(param.formViewId); + return formViewData; +} + +export async function formViewCreate(param: { + tableId: string; + body: FormReqType; +}) { + Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'form' }); + const view = await View.insert({ + ...param.body, + // todo: sanitize + fk_model_id: param.tableId, + type: ViewTypes.FORM, + }); + return view; +} + +// @ts-ignore +export async function formViewUpdate(param: { + formViewId: string; + body: FormReqType; +}) { + Tele.emit('evt', { evt_type: 'view:updated', type: 'grid' }); + await FormView.update(param.formViewId, param.body); +} diff --git a/packages/nocodb/src/lib/services/galleryViewService.ts b/packages/nocodb/src/lib/services/galleryViewService.ts new file mode 100644 index 0000000000..c66d8d655e --- /dev/null +++ b/packages/nocodb/src/lib/services/galleryViewService.ts @@ -0,0 +1,31 @@ +import { Request, Response, Router } from 'express'; +import { GalleryReqType, GalleryType, ViewTypes } from 'nocodb-sdk'; +import View from '../models/View'; +import GalleryView from '../models/GalleryView'; +import { Tele } from 'nc-help'; + +export async function galleryViewGet(param: { galleryViewId: string }) { + return await GalleryView.get(param.galleryViewId); +} + +export async function galleryViewCreate(param: { + tableId: string; + gallery: GalleryReqType; +}) { + Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'gallery' }); + const view = await View.insert({ + ...param.gallery, + // todo: sanitize + fk_model_id: param.tableId, + type: ViewTypes.GALLERY, + }); + return view; +} + +export async function galleryViewUpdate(param: { + galleryViewId: string; + gallery: GalleryReqType; +}) { + Tele.emit('evt', { evt_type: 'view:updated', type: 'gallery' }); + await GalleryView.update(param.galleryViewId, param.gallery); +} diff --git a/packages/nocodb/src/lib/services/gridViewService.ts b/packages/nocodb/src/lib/services/gridViewService.ts new file mode 100644 index 0000000000..55315d1fc8 --- /dev/null +++ b/packages/nocodb/src/lib/services/gridViewService.ts @@ -0,0 +1,26 @@ +import { Tele } from 'nc-help'; +import { GridReqType, ViewTypes } from 'nocodb-sdk'; +import { View } from '../models'; +import { GridView } from '../models'; + +export async function gridViewCreate(param: { + tableId: string; + grid: GridReqType; +}) { + const view = await View.insert({ + ...param.grid, + // todo: sanitize + fk_model_id: param.tableId, + type: ViewTypes.GRID, + }); + Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'grid' }); + return view; +} + +export async function gridViewUpdate(param: { + viewId: string; + grid: GridReqType; +}) { + Tele.emit('evt', { evt_type: 'view:updated', type: 'grid' }); + return await GridView.update(param.viewId, param.grid); +} diff --git a/packages/nocodb/src/lib/services/hookService.ts b/packages/nocodb/src/lib/services/hookService.ts index 36de0ceb15..17cef05229 100644 --- a/packages/nocodb/src/lib/services/hookService.ts +++ b/packages/nocodb/src/lib/services/hookService.ts @@ -1,30 +1,19 @@ import { Tele } from 'nc-help'; -import catchError from '../helpers/catchError'; -import { Request, Response, Router } from 'express'; import { Hook, Model } from '../models'; -import { HookListType, HookReqType, HookTestReqType, HookType, TableReqType } from 'nocodb-sdk' -import { PagedResponseImpl } from '../helpers/PagedResponse'; -import { invokeWebhook } from '../helpers/webhookHelpers'; -import populateSamplePayload from '../helpers/populateSamplePayload'; -import ncMetaAclMw from '../helpers/ncMetaAclMw'; -import { metaApiMetrics } from '../helpers/apiMetrics'; -import { getAjvValidatorMw } from './helpers'; +import { HookReqType, HookTestReqType } from 'nocodb-sdk'; -export async function hookList( -param: { - tableId: string; -} -) { +import { invokeWebhook } from '../meta/helpers/webhookHelpers'; +import populateSamplePayload from '../meta/helpers/populateSamplePayload'; + +export async function hookList(param: { tableId: string }) { // todo: pagination - await Hook.list({ fk_model_id: param.tableId })) + return await Hook.list({ fk_model_id: param.tableId }); } -export async function hookCreate( -param: { +export async function hookCreate(param: { tableId: string; - hook: HookReqType -} -) { + hook: HookReqType; +}) { Tele.emit('evt', { evt_type: 'webhooks:created' }); const hook = await Hook.insert({ ...param.hook, @@ -33,25 +22,16 @@ param: { return hook; } -export async function hookDelete( -param :{ - hookId: string; -} -) { +export async function hookDelete(param: { hookId: string }) { Tele.emit('evt', { evt_type: 'webhooks:deleted' }); await Hook.delete(param.hookId); return true; } -export async function hookUpdate( - param: { - hookId: string; - hook: HookReqType; - } -) { +export async function hookUpdate(param: { hookId: string; hook: HookReqType }) { Tele.emit('evt', { evt_type: 'webhooks:updated' }); - return await Hook.update(param.hookId, param.hook) + return await Hook.update(param.hookId, param.hook); } export async function hookTest(param: { @@ -75,9 +55,9 @@ export async function hookTest(param: { Tele.emit('evt', { evt_type: 'webhooks:tested' }); - return true + return true; } -export async function tableSampleData(param:{ +export async function tableSampleData(param: { tableId: string; operation: 'insert' | 'update'; }) { @@ -85,39 +65,3 @@ export async function tableSampleData(param:{ return await populateSamplePayload(model, false, param.operation); } - -const router = Router({ mergeParams: true }); -router.get( - '/api/v1/db/meta/tables/:tableId/hooks', - metaApiMetrics, - ncMetaAclMw(hookList, 'hookList') -); -router.post( - '/api/v1/db/meta/tables/:tableId/hooks/test', - metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/HookTestReq'), - ncMetaAclMw(hookTest, 'hookTest') -); -router.post( - '/api/v1/db/meta/tables/:tableId/hooks', - metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/HookReq'), - ncMetaAclMw(hookCreate, 'hookCreate') -); -router.delete( - '/api/v1/db/meta/hooks/:hookId', - metaApiMetrics, - ncMetaAclMw(hookDelete, 'hookDelete') -); -router.patch( - '/api/v1/db/meta/hooks/:hookId', - metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/HookReq'), - ncMetaAclMw(hookUpdate, 'hookUpdate') -); -router.get( - '/api/v1/db/meta/tables/:tableId/hooks/samplePayload/:operation', - metaApiMetrics, - catchError(tableSampleData) -); -export default router; diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 2d53935889..6d33b2a177 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -6,3 +6,11 @@ export * as sortService from './sortService'; export * as baseService from './baseService'; export * as apiTokenService from './apiTokenService'; export * as viewService from './viewService'; +export * as hookService from './hookService'; +export * as pluginService from './pluginService'; +export * as utilService from './utilService'; +export * as formViewService from './formViewService'; +export * as formViewColumnService from './formViewColumnService'; +export * as gridViewService from './gridViewService'; +export * as galleryViewService from './galleryViewService'; +export * as kanbanViewService from './kanbanViewService'; diff --git a/packages/nocodb/src/lib/services/kanbanViewService.ts b/packages/nocodb/src/lib/services/kanbanViewService.ts new file mode 100644 index 0000000000..6dc676e351 --- /dev/null +++ b/packages/nocodb/src/lib/services/kanbanViewService.ts @@ -0,0 +1,29 @@ +import { KanbanReqType, ViewTypes } from 'nocodb-sdk'; +import { KanbanView, View } from '../models'; +import { Tele } from 'nc-help'; + +export async function kanbanViewGet(param: { kanbanViewId: string }) { + return await KanbanView.get(param.kanbanViewId); +} + +export async function kanbanViewCreate(param: { + tableId: string; + kanban: KanbanReqType; +}) { + Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'kanban' }); + const view = await View.insert({ + ...param.kanban, + // todo: sanitize + fk_model_id: param.tableId, + type: ViewTypes.KANBAN, + }); + return view; +} + +export async function kanbanViewUpdate(param: { + kanbanViewId: string; + kanban: KanbanReqType; +}) { + Tele.emit('evt', { evt_type: 'view:updated', type: 'kanban' }); + return await KanbanView.update(param.kanbanViewId, param.kanban); +} diff --git a/packages/nocodb/src/lib/services/pluginService.ts b/packages/nocodb/src/lib/services/pluginService.ts new file mode 100644 index 0000000000..b468e7240c --- /dev/null +++ b/packages/nocodb/src/lib/services/pluginService.ts @@ -0,0 +1,30 @@ +import { Tele } from 'nc-help'; +import { Plugin } from '../models' +import { PluginTestReqType, PluginType } from 'nocodb-sdk' +import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; + +export async function pluginList() { + return await Plugin.list() +} + +export async function pluginTest(param:{body: PluginTestReqType}) { + Tele.emit('evt', { evt_type: 'plugin:tested' }); + return await NcPluginMgrv2.test(param.body) +} + +export async function pluginRead(param: { pluginId: string }) { + return await Plugin.get(param.pluginId) +} +export async function pluginUpdate( + param: { pluginId: string; plugin: PluginType } +) { + const plugin = await Plugin.update(param.pluginId, param.plugin); + Tele.emit('evt', { + evt_type: plugin.active ? 'plugin:installed' : 'plugin:uninstalled', + title: plugin.title, + }); + return plugin +} +export async function isPluginActive(param: { pluginTitle: string }) { + return await Plugin.isPluginActive(param.pluginTitle) +} diff --git a/packages/nocodb/src/lib/services/utilService.ts b/packages/nocodb/src/lib/services/utilService.ts new file mode 100644 index 0000000000..14c42094f1 --- /dev/null +++ b/packages/nocodb/src/lib/services/utilService.ts @@ -0,0 +1,381 @@ +import { compareVersions, validate } from 'compare-versions'; + +import { ViewTypes } from 'nocodb-sdk'; +import { Project } from '../models'; +import { NcError } from '../meta/helpers/catchError'; +import Noco from '../Noco'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import { MetaTable } from '../utils/globals'; +import { packageVersion } from '../utils/packageVersion'; +import SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2'; +import NcConfigFactory, { + defaultConnectionConfig, +} from '../utils/NcConfigFactory'; +import { User } from '../models'; +import axios from 'axios'; +import { NC_ATTACHMENT_FIELD_SIZE } from '../constants'; + +const versionCache = { + releaseVersion: null, + lastFetched: null, +}; + + + +export async function testConnection(param:{body: any}) { + return await SqlMgrv2.testConnection(param.body); +} + +export async function appInfo(param: { req: { ncSiteUrl: string } }) { + const projectHasAdmin = !(await User.isFirst()); + const result = { + authType: 'jwt', + projectHasAdmin, + firstUser: !projectHasAdmin, + type: 'rest', + env: process.env.NODE_ENV, + googleAuthEnabled: !!( + process.env.NC_GOOGLE_CLIENT_ID && process.env.NC_GOOGLE_CLIENT_SECRET + ), + githubAuthEnabled: !!( + process.env.NC_GITHUB_CLIENT_ID && process.env.NC_GITHUB_CLIENT_SECRET + ), + oneClick: !!process.env.NC_ONE_CLICK, + connectToExternalDB: !process.env.NC_CONNECT_TO_EXTERNAL_DB_DISABLED, + version: packageVersion, + defaultLimit: Math.max( + Math.min( + +process.env.DB_QUERY_LIMIT_DEFAULT || 25, + +process.env.DB_QUERY_LIMIT_MAX || 100 + ), + +process.env.DB_QUERY_LIMIT_MIN || 1 + ), + timezone: defaultConnectionConfig.timezone, + ncMin: !!process.env.NC_MIN, + teleEnabled: process.env.NC_DISABLE_TELE === 'true' ? false : true, + auditEnabled: process.env.NC_DISABLE_AUDIT === 'true' ? false : true, + ncSiteUrl: (param.req as any).ncSiteUrl, + ee: Noco.isEE(), + ncAttachmentFieldSize: NC_ATTACHMENT_FIELD_SIZE, + ncMaxAttachmentsAllowed: +(process.env.NC_MAX_ATTACHMENTS_ALLOWED || 10), + }; + + return result; +} + +export async function versionInfo() { + if ( + !versionCache.lastFetched || + (versionCache.lastFetched && + versionCache.lastFetched < Date.now() - 1000 * 60 * 60) + ) { + const nonBetaTags = await axios + .get('https://api.github.com/repos/nocodb/nocodb/tags', { + timeout: 5000, + }) + .then((response) => { + return response.data + .map((x) => x.name) + .filter( + (v) => validate(v) && !v.includes('finn') && !v.includes('beta') + ) + .sort((x, y) => compareVersions(y, x)); + }) + .catch(() => null); + if (nonBetaTags && nonBetaTags.length > 0) { + versionCache.releaseVersion = nonBetaTags[0]; + } + versionCache.lastFetched = Date.now(); + } + + const response = { + currentVersion: packageVersion, + releaseVersion: versionCache.releaseVersion, + }; + + return response; +} + +export async function appHealth() { + return { + message: 'OK', + timestamp: Date.now(), + uptime: process.uptime(), + }; +} + +async function _axiosRequestMake(param: { + body: { + apiMeta: any; + }; +}) { + const { apiMeta } = param.body; + + if (apiMeta?.body) { + try { + apiMeta.body = JSON.parse(apiMeta.body); + } catch (e) { + console.log(e); + } + } + + if (apiMeta?.auth) { + try { + apiMeta.auth = JSON.parse(apiMeta.auth); + } catch (e) { + console.log(e); + } + } + + apiMeta.response = {}; + const _req = { + params: apiMeta.parameters + ? apiMeta.parameters.reduce((paramsObj, param) => { + if (param.name && param.enabled) { + paramsObj[param.name] = param.value; + } + return paramsObj; + }, {}) + : {}, + url: apiMeta.url, + method: apiMeta.method || 'GET', + data: apiMeta.body || {}, + headers: apiMeta.headers + ? apiMeta.headers.reduce((headersObj, header) => { + if (header.name && header.enabled) { + headersObj[header.name] = header.value; + } + return headersObj; + }, {}) + : {}, + responseType: apiMeta.responseType || 'json', + withCredentials: true, + }; + const data = await require('axios')(_req); + return data?.data; +} + +export async function axiosRequestMake(param: { + body: { + apiMeta: any; + }; +}) { + const { + apiMeta: { url }, + } = param.body; + const isExcelImport = /.*\.(xls|xlsx|xlsm|ods|ots)/; + const isCSVImport = /.*\.(csv)/; + const ipBlockList = + /(10)(\.([2]([0-5][0-5]|[01234][6-9])|[1][0-9][0-9]|[1-9][0-9]|[0-9])){3}|(172)\.(1[6-9]|2[0-9]|3[0-1])(\.(2[0-4][0-9]|25[0-5]|[1][0-9][0-9]|[1-9][0-9]|[0-9])){2}|(192)\.(168)(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])){2}|(0.0.0.0)|localhost?/g; + if ( + ipBlockList.test(url) || + (!isCSVImport.test(url) && !isExcelImport.test(url)) + ) { + return {}; + } + if (isCSVImport || isExcelImport) { + param.body.apiMeta.responseType = 'arraybuffer'; + } + return await _axiosRequestMake({ + body: param.body, + }); +} + +export async function urlToDbConfig(param: { + body: { + url: string; + }; +}) { + const { url } = param.body; + try { + const connectionConfig = NcConfigFactory.extractXcUrlFromJdbc(url, true); + return connectionConfig; + } catch (error) { + return NcError.internalServerError(); + } +} + +interface ViewCount { + formCount: number | null; + gridCount: number | null; + galleryCount: number | null; + kanbanCount: number | null; + total: number | null; + sharedFormCount: number | null; + sharedGridCount: number | null; + sharedGalleryCount: number | null; + sharedKanbanCount: number | null; + sharedTotal: number | null; + sharedLockedCount: number | null; +} + +interface AllMeta { + projectCount: number; + projects: ( + | { + external?: boolean | null; + tableCount: { + table: number; + view: number; + } | null; + viewCount: ViewCount; + webhookCount: number | null; + filterCount: number | null; + sortCount: number | null; + rowCount: ({ totalRecords: number } | null)[] | null; + userCount: number | null; + } + | { error: string } + )[]; + userCount: number; + sharedBaseCount: number; +} + +export async function aggregatedMetaInfo() { + const [projects, userCount] = await Promise.all([ + Project.list({}), + Noco.ncMeta.metaCount(null, null, MetaTable.USERS), + ]); + + const result: AllMeta = { + projectCount: projects.length, + projects: [], + userCount, + sharedBaseCount: 0, + }; + + result.projects.push( + ...extractResultOrNull( + await Promise.allSettled( + projects.map(async (project) => { + if (project.uuid) result.sharedBaseCount++; + const [ + tableCount, + dbViewCount, + viewCount, + webhookCount, + filterCount, + sortCount, + rowCount, + userCount, + ] = extractResultOrNull( + await Promise.allSettled([ + // db tables count + Noco.ncMeta.metaCount(project.id, null, MetaTable.MODELS, { + condition: { + type: 'table', + }, + }), + // db views count + Noco.ncMeta.metaCount(project.id, null, MetaTable.MODELS, { + condition: { + type: 'view', + }, + }), + // views count + (async () => { + const views = await Noco.ncMeta.metaList2( + project.id, + null, + MetaTable.VIEWS + ); + // grid, form, gallery, kanban and shared count + return views.reduce( + (out, view) => { + out.total++; + + switch (view.type) { + case ViewTypes.GRID: + out.gridCount++; + if (view.uuid) out.sharedGridCount++; + break; + case ViewTypes.FORM: + out.formCount++; + if (view.uuid) out.sharedFormCount++; + break; + case ViewTypes.GALLERY: + out.galleryCount++; + if (view.uuid) out.sharedGalleryCount++; + break; + case ViewTypes.KANBAN: + out.kanbanCount++; + if (view.uuid) out.sharedKanbanCount++; + } + + if (view.uuid) { + if (view.password) out.sharedLockedCount++; + out.sharedTotal++; + } + + return out; + }, + { + formCount: 0, + gridCount: 0, + galleryCount: 0, + kanbanCount: 0, + total: 0, + sharedFormCount: 0, + sharedGridCount: 0, + sharedGalleryCount: 0, + sharedKanbanCount: 0, + sharedTotal: 0, + sharedLockedCount: 0, + } + ); + })(), + // webhooks count + Noco.ncMeta.metaCount(project.id, null, MetaTable.HOOKS), + // filters count + Noco.ncMeta.metaCount(project.id, null, MetaTable.FILTER_EXP), + // sorts count + Noco.ncMeta.metaCount(project.id, null, MetaTable.SORT), + // row count per base + project.getBases().then(async (bases) => { + return extractResultOrNull( + await Promise.allSettled( + bases.map(async (base) => + (await NcConnectionMgrv2.getSqlClient(base)) + .totalRecords?.() + ?.then((result) => result?.data) + ) + ) + ); + }), + // project users count + Noco.ncMeta.metaCount(null, null, MetaTable.PROJECT_USERS, { + condition: { + project_id: project.id, + }, + aggField: '*', + }), + ]) + ); + + return { + tableCount: { table: tableCount, view: dbViewCount }, + external: !project.is_meta, + viewCount, + webhookCount, + filterCount, + sortCount, + rowCount, + userCount, + }; + }) + ) + ) + ); + + return result; +} + +const extractResultOrNull = (results: PromiseSettledResult[]) => { + return results.map((result) => { + if (result.status === 'fulfilled') { + return result.value; + } + console.log(result.reason); + return null; + }); +}; diff --git a/packages/nocodb/src/lib/services/viewService.ts b/packages/nocodb/src/lib/services/viewService.ts index 4d8cf2f1bc..d89d2b91ee 100644 --- a/packages/nocodb/src/lib/services/viewService.ts +++ b/packages/nocodb/src/lib/services/viewService.ts @@ -3,7 +3,6 @@ import { Tele } from 'nc-help'; import { SharedViewReqType, ViewReqType } from 'nocodb-sdk'; import { xcVisibilityMetaGet } from '../meta/api/modelVisibilityApis'; - export async function viewList(param: { tableId: string; user: { From b0d8ee8eb19380b51a132fa95028ef96de9092d2 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 28 Feb 2023 19:25:18 +0530 Subject: [PATCH 07/64] refcator: handler to service-controller - grid column and view column (WIP) Signed-off-by: Pranav C --- .../controllers/gridViewColumnController.ts | 17 +++++--- .../lib/controllers/viewColumnController.ts | 31 +++++++-------- .../src/lib/services/galleryViewService.ts | 6 +-- .../src/lib/services/gridViewColumnService.ts | 15 +++++++ packages/nocodb/src/lib/services/index.ts | 2 + .../src/lib/services/viewColumnService.ts | 39 +++++++++++++++++++ 6 files changed, 84 insertions(+), 26 deletions(-) create mode 100644 packages/nocodb/src/lib/services/gridViewColumnService.ts create mode 100644 packages/nocodb/src/lib/services/viewColumnService.ts diff --git a/packages/nocodb/src/lib/controllers/gridViewColumnController.ts b/packages/nocodb/src/lib/controllers/gridViewColumnController.ts index f18a175c9e..3b535cce9e 100644 --- a/packages/nocodb/src/lib/controllers/gridViewColumnController.ts +++ b/packages/nocodb/src/lib/controllers/gridViewColumnController.ts @@ -1,17 +1,24 @@ import { Request, Response, Router } from 'express'; -import GridViewColumn from '../models/GridViewColumn'; -import { Tele } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import { getAjvValidatorMw } from '../meta/api/helpers'; +import { gridViewColumnService } from '../services'; export async function columnList(req: Request, res: Response) { - res.json(await GridViewColumn.list(req.params.gridViewId)); + res.json( + await gridViewColumnService.columnList({ + gridViewId: req.params.gridViewId, + }) + ); } export async function gridColumnUpdate(req: Request, res: Response) { - Tele.emit('evt', { evt_type: 'gridViewColumn:updated' }); - res.json(await GridViewColumn.update(req.params.gridViewColumnId, req.body)); + res.json( + await gridViewColumnService.gridColumnUpdate({ + gridViewColumnId: req.params.gridViewColumnId, + grid: req.body, + }) + ); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/viewColumnController.ts b/packages/nocodb/src/lib/controllers/viewColumnController.ts index a7c7fda7eb..aece1ca06e 100644 --- a/packages/nocodb/src/lib/controllers/viewColumnController.ts +++ b/packages/nocodb/src/lib/controllers/viewColumnController.ts @@ -1,33 +1,30 @@ import { Request, Response, Router } from 'express'; -import View from '../models/View'; -import { Tele } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { viewColumnService } from '../services'; export async function columnList(req: Request, res: Response) { - res.json(await View.getColumns(req.params.viewId)); + res.json(await viewColumnService.columnList({ viewId: req.params.viewId })); } + export async function columnAdd(req: Request, res: Response) { - const viewColumn = await View.insertOrUpdateColumn( - req.params.viewId, - req.body.fk_column_id, - { + const viewColumn = await viewColumnService.columnAdd({ + viewId: req.params.viewId, + columnId: req.body.fk_column_id, + column: { ...req.body, view_id: req.params.viewId, - } - ); - Tele.emit('evt', { evt_type: 'viewColumn:inserted' }); - + }, + }); res.json(viewColumn); } export async function columnUpdate(req: Request, res: Response) { - const result = await View.updateColumn( - req.params.viewId, - req.params.columnId, - req.body - ); - Tele.emit('evt', { evt_type: 'viewColumn:updated' }); + const result = await viewColumnService.columnUpdate({ + viewId: req.params.viewId, + columnId: req.params.columnId, + column: req.body, + }); res.json(result); } diff --git a/packages/nocodb/src/lib/services/galleryViewService.ts b/packages/nocodb/src/lib/services/galleryViewService.ts index c66d8d655e..9132202b74 100644 --- a/packages/nocodb/src/lib/services/galleryViewService.ts +++ b/packages/nocodb/src/lib/services/galleryViewService.ts @@ -1,8 +1,6 @@ -import { Request, Response, Router } from 'express'; -import { GalleryReqType, GalleryType, ViewTypes } from 'nocodb-sdk'; -import View from '../models/View'; -import GalleryView from '../models/GalleryView'; +import { GalleryReqType, ViewTypes } from 'nocodb-sdk'; import { Tele } from 'nc-help'; +import { GalleryView, View } from '../models' export async function galleryViewGet(param: { galleryViewId: string }) { return await GalleryView.get(param.galleryViewId); diff --git a/packages/nocodb/src/lib/services/gridViewColumnService.ts b/packages/nocodb/src/lib/services/gridViewColumnService.ts new file mode 100644 index 0000000000..06731ec3b1 --- /dev/null +++ b/packages/nocodb/src/lib/services/gridViewColumnService.ts @@ -0,0 +1,15 @@ +import { GridColumnReqType } from 'nocodb-sdk'; +import GridViewColumn from '../models/GridViewColumn'; +import { Tele } from 'nc-help'; + +export async function columnList(param: { gridViewId: string }) { + return await GridViewColumn.list(param.gridViewId); +} + +export async function gridColumnUpdate(param: { + gridViewColumnId: string; + grid: GridColumnReqType; +}) { + Tele.emit('evt', { evt_type: 'gridViewColumn:updated' }); + return await GridViewColumn.update(param.gridViewColumnId, param.grid); +} diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 6d33b2a177..d4998f493b 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -14,3 +14,5 @@ export * as formViewColumnService from './formViewColumnService'; export * as gridViewService from './gridViewService'; export * as galleryViewService from './galleryViewService'; export * as kanbanViewService from './kanbanViewService'; +export * as gridViewColumnService from './gridViewColumnService'; +export * as viewColumnService from './viewColumnService'; diff --git a/packages/nocodb/src/lib/services/viewColumnService.ts b/packages/nocodb/src/lib/services/viewColumnService.ts new file mode 100644 index 0000000000..f6825691ad --- /dev/null +++ b/packages/nocodb/src/lib/services/viewColumnService.ts @@ -0,0 +1,39 @@ +import { Tele } from 'nc-help'; +import { View } from '../models' + +export async function columnList(param: { viewId: string }) { + return await View.getColumns(param.viewId); +} +export async function columnAdd(param: { + viewId: string; + columnId: string; + // todo: add proper type for grid column in swagger + column: any; +}) { + const viewColumn = await View.insertOrUpdateColumn( + param.viewId, + param.columnId, + { + ...param.column, + view_id: param.viewId, + } + ); + Tele.emit('evt', { evt_type: 'viewColumn:inserted' }); + + return viewColumn; +} + +export async function columnUpdate(param: { + viewId: string; + columnId: string; + // todo: add proper type for grid column in swagger + column: any; +}) { + const result = await View.updateColumn( + param.viewId, + param.columnId, + param.column + ); + Tele.emit('evt', { evt_type: 'viewColumn:updated' }); + return result; +} From 3af09aacb2221a7598d200065a71de1f2d58f375 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Mar 2023 01:06:49 +0530 Subject: [PATCH 08/64] refcator: handler to service-controller - metadiff and map view (WIP) Signed-off-by: Pranav C --- .../src/lib/controllers/mapViewController.ts | 27 +- .../src/lib/controllers/metaDiffController.ts | 1091 +--------------- .../src/lib/meta/api/helpers/populateMeta.ts | 2 +- packages/nocodb/src/lib/models/Model.ts | 578 ++++----- packages/nocodb/src/lib/services/index.ts | 2 + .../nocodb/src/lib/services/mapViewService.ts | 34 + .../src/lib/services/metaDiffService.ts | 1105 +++++++++++++++++ 7 files changed, 1459 insertions(+), 1380 deletions(-) create mode 100644 packages/nocodb/src/lib/services/mapViewService.ts create mode 100644 packages/nocodb/src/lib/services/metaDiffService.ts diff --git a/packages/nocodb/src/lib/controllers/mapViewController.ts b/packages/nocodb/src/lib/controllers/mapViewController.ts index 9c5123f98a..3a982d1b03 100644 --- a/packages/nocodb/src/lib/controllers/mapViewController.ts +++ b/packages/nocodb/src/lib/controllers/mapViewController.ts @@ -1,29 +1,30 @@ import { Request, Response, Router } from 'express'; -import { MapType, ViewTypes } from 'nocodb-sdk'; -import View from '../models/View'; +import { MapType } from 'nocodb-sdk'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { Tele } from 'nc-help'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import MapView from '../models/MapView'; +import { mapViewService } from '../services'; export async function mapViewGet(req: Request, res: Response) { - res.json(await MapView.get(req.params.mapViewId)); + res.json( + await mapViewService.mapViewGet({ mapViewId: req.params.mapViewId }) + ); } export async function mapViewCreate(req: Request, res) { - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'map' }); - const view = await View.insert({ - ...req.body, - // todo: sanitize - fk_model_id: req.params.tableId, - type: ViewTypes.MAP, + const view = await mapViewService.mapViewCreate({ + tableId: req.params.tableId, + map: req.body, }); res.json(view); } export async function mapViewUpdate(req, res) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'map' }); - res.json(await MapView.update(req.params.mapViewId, req.body)); + res.json( + await mapViewService.mapViewUpdate({ + mapViewId: req.params.mapViewId, + map: req.body, + }) + ); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/metaDiffController.ts b/packages/nocodb/src/lib/controllers/metaDiffController.ts index 551846074d..45ac703a0d 100644 --- a/packages/nocodb/src/lib/controllers/metaDiffController.ts +++ b/packages/nocodb/src/lib/controllers/metaDiffController.ts @@ -1,1100 +1,35 @@ -// // Project CRUD - -import { Tele } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import Model from '../models/Model'; -import Project from '../models/Project'; -import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; -import { isVirtualCol, ModelTypes, RelationTypes, UITypes } from 'nocodb-sdk'; import { Router } from 'express'; -import Base from '../models/Base'; -import ModelXcMetaFactory from '../db/sql-mgr/code/models/xc/ModelXcMetaFactory'; -import Column from '../models/Column'; -import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; -import { getUniqueColumnAliasName } from '../meta/helpers/getUniqueName'; -import NcHelp from '../utils/NcHelp'; -import getTableNameAlias, { getColumnNameAlias } from '../meta/helpers/getTableName'; -import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; -import getColumnUiType from '../meta/helpers/getColumnUiType'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; - -export enum MetaDiffType { - TABLE_NEW = 'TABLE_NEW', - TABLE_REMOVE = 'TABLE_REMOVE', - TABLE_COLUMN_ADD = 'TABLE_COLUMN_ADD', - TABLE_COLUMN_TYPE_CHANGE = 'TABLE_COLUMN_TYPE_CHANGE', - TABLE_COLUMN_REMOVE = 'TABLE_COLUMN_REMOVE', - VIEW_NEW = 'VIEW_NEW', - VIEW_REMOVE = 'VIEW_REMOVE', - VIEW_COLUMN_ADD = 'VIEW_COLUMN_ADD', - VIEW_COLUMN_TYPE_CHANGE = 'VIEW_COLUMN_TYPE_CHANGE', - VIEW_COLUMN_REMOVE = 'VIEW_COLUMN_REMOVE', - TABLE_RELATION_ADD = 'TABLE_RELATION_ADD', - TABLE_RELATION_REMOVE = 'TABLE_RELATION_REMOVE', - TABLE_VIRTUAL_M2M_REMOVE = 'TABLE_VIRTUAL_M2M_REMOVE', -} - -const applyChangesPriorityOrder = [ - MetaDiffType.VIEW_COLUMN_REMOVE, - MetaDiffType.TABLE_RELATION_REMOVE, -]; - -type MetaDiff = { - title?: string; - table_name: string; - base_id: string; - type: ModelTypes; - meta?: any; - detectedChanges: Array; -}; - -type MetaDiffChange = { - msg?: string; - // type: MetaDiffType; -} & ( - | { - type: MetaDiffType.TABLE_NEW | MetaDiffType.VIEW_NEW; - tn?: string; - } - | { - type: MetaDiffType.TABLE_REMOVE | MetaDiffType.VIEW_REMOVE; - tn?: string; - model?: Model; - id?: string; - } - | { - type: MetaDiffType.TABLE_COLUMN_ADD | MetaDiffType.VIEW_COLUMN_ADD; - tn?: string; - model?: Model; - id?: string; - cn: string; - } - | { - type: - | MetaDiffType.TABLE_COLUMN_TYPE_CHANGE - | MetaDiffType.VIEW_COLUMN_TYPE_CHANGE - | MetaDiffType.TABLE_COLUMN_REMOVE - | MetaDiffType.VIEW_COLUMN_REMOVE; - tn?: string; - model?: Model; - id?: string; - cn: string; - column: Column; - colId?: string; - } - | { - type: MetaDiffType.TABLE_RELATION_REMOVE; - tn?: string; - rtn?: string; - cn?: string; - rcn?: string; - colId: string; - column: Column; - } - | { - type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE; - tn?: string; - rtn?: string; - cn?: string; - rcn?: string; - colId: string; - column: Column; - } - | { - type: MetaDiffType.TABLE_RELATION_ADD; - tn?: string; - rtn?: string; - cn?: string; - rcn?: string; - relationType: RelationTypes; - cstn?: string; - } -); - -async function getMetaDiff( - sqlClient, - project: Project, - base: Base -): Promise> { - const changes: Array = []; - const virtualRelationColumns: Column[] = []; - - // @ts-ignore - const tableList: Array<{ tn: string }> = ( - await sqlClient.tableList() - )?.data?.list?.filter((t) => { - if (project?.prefix && base.is_meta) { - return t.tn?.startsWith(project?.prefix); - } - return true; - }); - - const colListRef = {}; - const oldMetas = await base.getModels(); - // @ts-ignore - const oldTableMetas: Model[] = []; - const oldViewMetas: Model[] = []; - - for (const model of oldMetas) { - if (model.type === ModelTypes.TABLE) oldTableMetas.push(model); - else if (model.type === ModelTypes.VIEW) oldViewMetas.push(model); - } - - // @ts-ignore - const relationList: Array<{ - tn: string; - rtn: string; - cn: string; - rcn: string; - found?: any; - cstn?: string; - }> = (await sqlClient.relationListAll())?.data?.list; - - for (const table of tableList) { - if (table.tn === 'nc_evolutions') continue; - - const oldMetaIdx = oldTableMetas.findIndex( - (m) => m.table_name === table.tn - ); - - // new table - if (oldMetaIdx === -1) { - changes.push({ - table_name: table.tn, - base_id: base.id, - type: ModelTypes.TABLE, - detectedChanges: [ - { - type: MetaDiffType.TABLE_NEW, - msg: `New table`, - }, - ], - }); - continue; - } - - const oldMeta = oldTableMetas[oldMetaIdx]; - - oldTableMetas.splice(oldMetaIdx, 1); - - const tableProp: MetaDiff = { - title: oldMeta.title, - meta: oldMeta.meta, - table_name: table.tn, - base_id: base.id, - type: ModelTypes.TABLE, - detectedChanges: [], - }; - changes.push(tableProp); - - // check for column change - colListRef[table.tn] = ( - await sqlClient.columnList({ tn: table.tn }) - )?.data?.list; - - await oldMeta.getColumns(); - - for (const column of colListRef[table.tn]) { - const oldColIdx = oldMeta.columns.findIndex( - (c) => c.column_name === column.cn - ); - - // new table - if (oldColIdx === -1) { - tableProp.detectedChanges.push({ - type: MetaDiffType.TABLE_COLUMN_ADD, - msg: `New column(${column.cn})`, - cn: column.cn, - id: oldMeta.id, - }); - continue; - } - - const [oldCol] = oldMeta.columns.splice(oldColIdx, 1); - - if (oldCol.dt !== column.dt) { - tableProp.detectedChanges.push({ - type: MetaDiffType.TABLE_COLUMN_TYPE_CHANGE, - msg: `Column type changed(${column.cn})`, - cn: oldCol.column_name, - id: oldMeta.id, - column: oldCol, - }); - } - } - for (const column of oldMeta.columns) { - if ( - [ - UITypes.LinkToAnotherRecord, - UITypes.Rollup, - UITypes.Lookup, - UITypes.Formula, - ].includes(column.uidt) - ) { - if (column.uidt === UITypes.LinkToAnotherRecord) { - virtualRelationColumns.push(column); - } - - continue; - } - - tableProp.detectedChanges.push({ - type: MetaDiffType.TABLE_COLUMN_REMOVE, - msg: `Column removed(${column.column_name})`, - cn: column.column_name, - id: oldMeta.id, - column: column, - colId: column.id, - }); - } - } - - for (const model of oldTableMetas) { - changes.push({ - table_name: model.table_name, - meta: model.meta, - base_id: base.id, - type: ModelTypes.TABLE, - detectedChanges: [ - { - type: MetaDiffType.TABLE_REMOVE, - msg: `Table removed`, - tn: model.table_name, - id: model.id, - model, - }, - ], - }); - } - - for (const relationCol of virtualRelationColumns) { - const colOpt = await relationCol.getColOptions(); - const parentCol = await colOpt.getParentColumn(); - const childCol = await colOpt.getChildColumn(); - const parentModel = await parentCol.getModel(); - const childModel = await childCol.getModel(); - - // many to many relation - if (colOpt.type === RelationTypes.MANY_TO_MANY) { - const m2mModel = await colOpt.getMMModel(); - - const relatedTable = tableList.find( - (t) => t.tn === parentModel.table_name - ); - const m2mTable = tableList.find((t) => t.tn === m2mModel.table_name); - - if (!relatedTable) { - changes - .find((t) => t.table_name === childModel.table_name) - .detectedChanges.push({ - type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, - msg: `Many to many removed(${relatedTable.tn} removed)`, - colId: relationCol.id, - column: relationCol, - }); - continue; - } - if (!m2mTable) { - changes - .find((t) => t.table_name === childModel.table_name) - .detectedChanges.push({ - type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, - msg: `Many to many removed(${m2mModel.table_name} removed)`, - colId: relationCol.id, - column: relationCol, - }); - continue; - } - - // verify columns - - const cColumns = (colListRef[childModel.table_name] = - colListRef[childModel.table_name] || - (await sqlClient.columnList({ tn: childModel.table_name }))?.data - ?.list); - - const pColumns = (colListRef[parentModel.table_name] = - colListRef[parentModel.table_name] || - (await sqlClient.columnList({ tn: parentModel.table_name }))?.data - ?.list); - - const vColumns = (colListRef[m2mTable.tn] = - colListRef[m2mTable.tn] || - (await sqlClient.columnList({ tn: m2mTable.tn }))?.data?.list); - - const m2mChildCol = await colOpt.getMMChildColumn(); - const m2mParentCol = await colOpt.getMMParentColumn(); - - if ( - pColumns.every((c) => c.cn !== parentCol.column_name) || - cColumns.every((c) => c.cn !== childCol.column_name) || - vColumns.every((c) => c.cn !== m2mChildCol.column_name) || - vColumns.every((c) => c.cn !== m2mParentCol.column_name) - ) { - changes - .find((t) => t.table_name === childModel.table_name) - .detectedChanges.push({ - type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, - msg: `Many to many removed(One of the relation column removed)`, - colId: relationCol.id, - column: relationCol, - }); - } - - continue; - } - - if (relationCol.colOptions.virtual) continue; - - const dbRelation = relationList.find( - (r) => - r.cn === childCol.column_name && - r.tn === childModel.table_name && - r.rcn === parentCol.column_name && - r.rtn === parentModel.table_name - ); - - if (dbRelation) { - dbRelation.found = dbRelation.found || {}; - - if (dbRelation.found[colOpt.type]) { - // todo: handle duplicate - } else { - dbRelation.found[colOpt.type] = true; - } - } else { - changes - .find( - (t) => - t.table_name === - (colOpt.type === RelationTypes.BELONGS_TO - ? childModel.table_name - : parentModel.table_name) - ) - .detectedChanges.push({ - type: MetaDiffType.TABLE_RELATION_REMOVE, - tn: childModel.table_name, - rtn: parentModel.table_name, - cn: childCol.column_name, - rcn: parentCol.column_name, - msg: `Relation removed`, - colId: relationCol.id, - column: relationCol, - }); - } - } - - for (const relation of relationList) { - if (!relation?.found?.[RelationTypes.BELONGS_TO]) { - changes - .find((t) => t.table_name === relation.tn) - ?.detectedChanges.push({ - type: MetaDiffType.TABLE_RELATION_ADD, - tn: relation.tn, - rtn: relation.rtn, - cn: relation.cn, - rcn: relation.rcn, - msg: `New relation added`, - relationType: RelationTypes.BELONGS_TO, - cstn: relation.cstn, - }); - } - if (!relation?.found?.[RelationTypes.HAS_MANY]) { - changes - .find((t) => t.table_name === relation.rtn) - ?.detectedChanges.push({ - type: MetaDiffType.TABLE_RELATION_ADD, - tn: relation.tn, - rtn: relation.rtn, - cn: relation.cn, - rcn: relation.rcn, - msg: `New relation added`, - relationType: RelationTypes.HAS_MANY, - }); - } - } - - // views - // @ts-ignore - const viewList: Array<{ - view_name: string; - tn: string; - type: 'view'; - }> = (await sqlClient.viewList())?.data?.list - ?.map((v) => { - v.type = 'view'; - v.tn = v.view_name; - return v; - }) - .filter((t) => { - if (project?.prefix && base.is_meta) { - return t.tn?.startsWith(project?.prefix); - } - return true; - }); // @ts-ignore - - for (const view of viewList) { - const oldMetaIdx = oldViewMetas.findIndex((m) => m.table_name === view.tn); - - // new table - if (oldMetaIdx === -1) { - changes.push({ - table_name: view.tn, - base_id: base.id, - type: ModelTypes.VIEW, - detectedChanges: [ - { - type: MetaDiffType.VIEW_NEW, - msg: `New view`, - }, - ], - }); - continue; - } - - const oldMeta = oldViewMetas[oldMetaIdx]; - - oldViewMetas.splice(oldMetaIdx, 1); - - const tableProp: MetaDiff = { - title: oldMeta.title, - meta: oldMeta.meta, - table_name: view.tn, - base_id: base.id, - type: ModelTypes.VIEW, - detectedChanges: [], - }; - changes.push(tableProp); - - // check for column change - colListRef[view.tn] = ( - await sqlClient.columnList({ tn: view.tn }) - )?.data?.list; - - await oldMeta.getColumns(); - - for (const column of colListRef[view.tn]) { - const oldColIdx = oldMeta.columns.findIndex( - (c) => c.column_name === column.cn - ); - - // new table - if (oldColIdx === -1) { - tableProp.detectedChanges.push({ - type: MetaDiffType.VIEW_COLUMN_ADD, - msg: `New column(${column.cn})`, - cn: column.cn, - id: oldMeta.id, - }); - continue; - } - - const [oldCol] = oldMeta.columns.splice(oldColIdx, 1); - - if (oldCol.dt !== column.dt) { - tableProp.detectedChanges.push({ - type: MetaDiffType.TABLE_COLUMN_TYPE_CHANGE, - msg: `Column type changed(${column.cn})`, - cn: oldCol.column_name, - id: oldMeta.id, - column: oldCol, - }); - } - } - for (const column of oldMeta.columns) { - if ( - [ - UITypes.LinkToAnotherRecord, - UITypes.Rollup, - UITypes.Lookup, - UITypes.Formula, - ].includes(column.uidt) - ) { - continue; - } - - tableProp.detectedChanges.push({ - type: MetaDiffType.VIEW_COLUMN_REMOVE, - msg: `Column removed(${column.column_name})`, - cn: column.column_name, - id: oldMeta.id, - column: column, - colId: column.id, - }); - } - } - - for (const model of oldViewMetas) { - changes.push({ - table_name: model.table_name, - meta: model.meta, - base_id: base.id, - type: ModelTypes.TABLE, - detectedChanges: [ - { - type: MetaDiffType.VIEW_REMOVE, - msg: `Table removed`, - tn: model.table_name, - id: model.id, - model, - }, - ], - }); - } - - return changes; -} +import { metaDiffService } from '../services'; export async function metaDiff(req, res) { - const project = await Project.getWithInfo(req.params.projectId); - let changes = []; - for (const base of project.bases) { - try { - // @ts-ignore - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - changes = changes.concat(await getMetaDiff(sqlClient, project, base)); - } catch (e) { - console.log(e); - } - } - - res.json(changes); + res.json(await metaDiffService.metaDiff({ projectId: req.params.projectId })); } export async function baseMetaDiff(req, res) { - const project = await Project.getWithInfo(req.params.projectId); - const base = await Base.get(req.params.baseId); - let changes = []; - - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - changes = await getMetaDiff(sqlClient, project, base); - - res.json(changes); + res.json( + await metaDiffService.baseMetaDiff({ + baseId: req.params.baseId, + projectId: req.params.projectId, + }) + ); } export async function metaDiffSync(req, res) { - const project = await Project.getWithInfo(req.params.projectId); - for (const base of project.bases) { - const virtualColumnInsert: Array<() => Promise> = []; - - // @ts-ignore - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - const changes = await getMetaDiff(sqlClient, project, base); - - /* Get all relations */ - // const relations = (await sqlClient.relationListAll())?.data?.list; - - for (const { table_name, detectedChanges } of changes) { - // reorder changes to apply relation remove changes - // before column remove to avoid foreign key constraint error - detectedChanges.sort((a, b) => { - return ( - applyChangesPriorityOrder.indexOf(b.type) - - applyChangesPriorityOrder.indexOf(a.type) - ); - }); - - for (const change of detectedChanges) { - switch (change.type) { - case MetaDiffType.TABLE_NEW: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - - mapDefaultDisplayValue(columns); - - const model = await Model.insert(project.id, base.id, { - table_name: table_name, - title: getTableNameAlias( - table_name, - base.is_meta ? project.prefix : '', - base - ), - type: ModelTypes.TABLE, - }); - - for (const column of columns) { - await Column.insert({ - uidt: getColumnUiType(base, column), - fk_model_id: model.id, - ...column, - title: getColumnNameAlias(column.column_name, base), - }); - } - } - break; - case MetaDiffType.VIEW_NEW: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - - mapDefaultDisplayValue(columns); - - const model = await Model.insert(project.id, base.id, { - table_name: table_name, - title: getTableNameAlias(table_name, project.prefix, base), - type: ModelTypes.VIEW, - }); - - for (const column of columns) { - await Column.insert({ - uidt: getColumnUiType(base, column), - fk_model_id: model.id, - ...column, - title: getColumnNameAlias(column.column_name, base), - }); - } - } - break; - case MetaDiffType.TABLE_REMOVE: - case MetaDiffType.VIEW_REMOVE: - { - await change.model.delete(); - } - break; - case MetaDiffType.TABLE_COLUMN_ADD: - case MetaDiffType.VIEW_COLUMN_ADD: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - const column = columns.find((c) => c.cn === change.cn); - column.uidt = getColumnUiType(base, column); - //todo: inflection - column.title = getColumnNameAlias(column.cn, base); - await Column.insert({ fk_model_id: change.id, ...column }); - } - // update old - // populateParams.tableNames.push({ tn }); - // populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn); - - break; - case MetaDiffType.TABLE_COLUMN_TYPE_CHANGE: - case MetaDiffType.VIEW_COLUMN_TYPE_CHANGE: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - const column = columns.find((c) => c.cn === change.cn); - const metaFact = ModelXcMetaFactory.create( - { client: base.type }, - {} - ); - column.uidt = metaFact.getUIDataType(column); - column.title = change.column.title; - await Column.update(change.column.id, column); - } - break; - case MetaDiffType.TABLE_COLUMN_REMOVE: - case MetaDiffType.VIEW_COLUMN_REMOVE: - await change.column.delete(); - break; - case MetaDiffType.TABLE_RELATION_REMOVE: - case MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE: - await change.column.delete(); - break; - case MetaDiffType.TABLE_RELATION_ADD: - { - virtualColumnInsert.push(async () => { - const parentModel = await Model.getByIdOrName({ - project_id: base.project_id, - base_id: base.id, - table_name: change.rtn, - }); - const childModel = await Model.getByIdOrName({ - project_id: base.project_id, - base_id: base.id, - table_name: change.tn, - }); - const parentCol = await parentModel - .getColumns() - .then((cols) => - cols.find((c) => c.column_name === change.rcn) - ); - const childCol = await childModel - .getColumns() - .then((cols) => - cols.find((c) => c.column_name === change.cn) - ); - - await Column.update(childCol.id, { - ...childCol, - uidt: UITypes.ForeignKey, - system: true, - }); - - if (change.relationType === RelationTypes.BELONGS_TO) { - const title = getUniqueColumnAliasName( - childModel.columns, - `${parentModel.title || parentModel.table_name}` - ); - await Column.insert({ - uidt: UITypes.LinkToAnotherRecord, - title, - fk_model_id: childModel.id, - fk_related_model_id: parentModel.id, - type: RelationTypes.BELONGS_TO, - fk_parent_column_id: parentCol.id, - fk_child_column_id: childCol.id, - virtual: false, - fk_index_name: change.cstn, - }); - } else if (change.relationType === RelationTypes.HAS_MANY) { - const title = getUniqueColumnAliasName( - childModel.columns, - `${childModel.title || childModel.table_name} List` - ); - await Column.insert({ - uidt: UITypes.LinkToAnotherRecord, - title, - fk_model_id: parentModel.id, - fk_related_model_id: childModel.id, - type: RelationTypes.HAS_MANY, - fk_parent_column_id: parentCol.id, - fk_child_column_id: childCol.id, - virtual: false, - fk_index_name: change.cstn, - }); - } - }); - } - break; - } - } - } - - await NcHelp.executeOperations(virtualColumnInsert, base.type); - - // populate m2m relations - await extractAndGenerateManyToManyRelations(await base.getModels()); - } - - Tele.emit('evt', { evt_type: 'metaDiff:synced' }); - + await metaDiffService.metaDiffSync({ projectId: req.params.projectId }); res.json({ msg: 'success' }); } export async function baseMetaDiffSync(req, res) { - const project = await Project.getWithInfo(req.params.projectId); - const base = await Base.get(req.params.baseId); - - const virtualColumnInsert: Array<() => Promise> = []; - - // @ts-ignore - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - const changes = await getMetaDiff(sqlClient, project, base); - - /* Get all relations */ - // const relations = (await sqlClient.relationListAll())?.data?.list; - - for (const { table_name, detectedChanges } of changes) { - for (const change of detectedChanges) { - switch (change.type) { - case MetaDiffType.TABLE_NEW: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - - mapDefaultDisplayValue(columns); - - const model = await Model.insert(project.id, base.id, { - table_name: table_name, - title: getTableNameAlias( - table_name, - base.is_meta ? project.prefix : '', - base - ), - type: ModelTypes.TABLE, - }); - - for (const column of columns) { - await Column.insert({ - uidt: getColumnUiType(base, column), - fk_model_id: model.id, - ...column, - title: getColumnNameAlias(column.column_name, base), - }); - } - } - break; - case MetaDiffType.VIEW_NEW: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - - mapDefaultDisplayValue(columns); - - const model = await Model.insert(project.id, base.id, { - table_name: table_name, - title: getTableNameAlias(table_name, project.prefix, base), - type: ModelTypes.VIEW, - }); - - for (const column of columns) { - await Column.insert({ - uidt: getColumnUiType(base, column), - fk_model_id: model.id, - ...column, - title: getColumnNameAlias(column.column_name, base), - }); - } - } - break; - case MetaDiffType.TABLE_REMOVE: - case MetaDiffType.VIEW_REMOVE: - { - await change.model.delete(); - } - break; - case MetaDiffType.TABLE_COLUMN_ADD: - case MetaDiffType.VIEW_COLUMN_ADD: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - const column = columns.find((c) => c.cn === change.cn); - column.uidt = getColumnUiType(base, column); - //todo: inflection - column.title = getColumnNameAlias(column.cn, base); - await Column.insert({ fk_model_id: change.id, ...column }); - } - // update old - // populateParams.tableNames.push({ tn }); - // populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn); - - break; - case MetaDiffType.TABLE_COLUMN_TYPE_CHANGE: - case MetaDiffType.VIEW_COLUMN_TYPE_CHANGE: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - const column = columns.find((c) => c.cn === change.cn); - const metaFact = ModelXcMetaFactory.create( - { client: base.type }, - {} - ); - column.uidt = metaFact.getUIDataType(column); - column.title = change.column.title; - await Column.update(change.column.id, column); - } - break; - case MetaDiffType.TABLE_COLUMN_REMOVE: - case MetaDiffType.VIEW_COLUMN_REMOVE: - await change.column.delete(); - break; - case MetaDiffType.TABLE_RELATION_REMOVE: - case MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE: - await change.column.delete(); - break; - case MetaDiffType.TABLE_RELATION_ADD: - { - virtualColumnInsert.push(async () => { - const parentModel = await Model.getByIdOrName({ - project_id: base.project_id, - base_id: base.id, - table_name: change.rtn, - }); - const childModel = await Model.getByIdOrName({ - project_id: base.project_id, - base_id: base.id, - table_name: change.tn, - }); - const parentCol = await parentModel - .getColumns() - .then((cols) => cols.find((c) => c.column_name === change.rcn)); - const childCol = await childModel - .getColumns() - .then((cols) => cols.find((c) => c.column_name === change.cn)); - - await Column.update(childCol.id, { - ...childCol, - uidt: UITypes.ForeignKey, - system: true, - }); - - if (change.relationType === RelationTypes.BELONGS_TO) { - const title = getUniqueColumnAliasName( - childModel.columns, - `${parentModel.title || parentModel.table_name}` - ); - await Column.insert({ - uidt: UITypes.LinkToAnotherRecord, - title, - fk_model_id: childModel.id, - fk_related_model_id: parentModel.id, - type: RelationTypes.BELONGS_TO, - fk_parent_column_id: parentCol.id, - fk_child_column_id: childCol.id, - virtual: false, - }); - } else if (change.relationType === RelationTypes.HAS_MANY) { - const title = getUniqueColumnAliasName( - childModel.columns, - `${childModel.title || childModel.table_name} List` - ); - await Column.insert({ - uidt: UITypes.LinkToAnotherRecord, - title, - fk_model_id: parentModel.id, - fk_related_model_id: childModel.id, - type: RelationTypes.HAS_MANY, - fk_parent_column_id: parentCol.id, - fk_child_column_id: childCol.id, - virtual: false, - }); - } - }); - } - break; - } - } - } - - await NcHelp.executeOperations(virtualColumnInsert, base.type); - - // populate m2m relations - await extractAndGenerateManyToManyRelations(await base.getModels()); - - Tele.emit('evt', { evt_type: 'baseMetaDiff:synced' }); + await metaDiffService.baseMetaDiffSync({ + projectId: req.params.projectId, + baseId: req.params.baseId, + }); res.json({ msg: 'success' }); } -async function isMMRelationExist( - model: Model, - assocModel: Model, - belongsToCol: Column -) { - let isExist = false; - const colChildOpt = - await belongsToCol.getColOptions(); - for (const col of await model.getColumns()) { - if (col.uidt === UITypes.LinkToAnotherRecord) { - const colOpt = await col.getColOptions(); - if ( - colOpt && - colOpt.type === RelationTypes.MANY_TO_MANY && - colOpt.fk_mm_model_id === assocModel.id && - colOpt.fk_child_column_id === colChildOpt.fk_parent_column_id && - colOpt.fk_mm_child_column_id === colChildOpt.fk_child_column_id - ) { - isExist = true; - break; - } - } - } - return isExist; -} - -// @ts-ignore -export async function extractAndGenerateManyToManyRelations( - modelsArr: Array -) { - for (const assocModel of modelsArr) { - await assocModel.getColumns(); - // check if table is a Bridge table(or Associative Table) by checking - // number of foreign keys and columns - - const normalColumns = assocModel.columns.filter((c) => !isVirtualCol(c)); - const belongsToCols: Column[] = []; - for (const col of assocModel.columns) { - if (col.uidt == UITypes.LinkToAnotherRecord) { - const colOpt = await col.getColOptions(); - if (colOpt?.type === RelationTypes.BELONGS_TO) belongsToCols.push(col); - } - } - - // todo: impl better method to identify m2m relation - if (belongsToCols?.length === 2 && normalColumns.length < 5) { - const modelA = await belongsToCols[0].colOptions.getRelatedTable(); - const modelB = await belongsToCols[1].colOptions.getRelatedTable(); - - await modelA.getColumns(); - await modelB.getColumns(); - - // check tableA already have the relation or not - const isRelationAvailInA = await isMMRelationExist( - modelA, - assocModel, - belongsToCols[0] - ); - const isRelationAvailInB = await isMMRelationExist( - modelB, - assocModel, - belongsToCols[1] - ); - - if (!isRelationAvailInA) { - await Column.insert({ - title: getUniqueColumnAliasName( - modelA.columns, - `${modelB.title} List` - ), - fk_model_id: modelA.id, - fk_related_model_id: modelB.id, - fk_mm_model_id: assocModel.id, - fk_child_column_id: belongsToCols[0].colOptions.fk_parent_column_id, - fk_parent_column_id: belongsToCols[1].colOptions.fk_parent_column_id, - fk_mm_child_column_id: belongsToCols[0].colOptions.fk_child_column_id, - fk_mm_parent_column_id: - belongsToCols[1].colOptions.fk_child_column_id, - type: RelationTypes.MANY_TO_MANY, - uidt: UITypes.LinkToAnotherRecord, - }); - } - if (!isRelationAvailInB) { - await Column.insert({ - title: getUniqueColumnAliasName( - modelB.columns, - `${modelA.title} List` - ), - fk_model_id: modelB.id, - fk_related_model_id: modelA.id, - fk_mm_model_id: assocModel.id, - fk_child_column_id: belongsToCols[1].colOptions.fk_parent_column_id, - fk_parent_column_id: belongsToCols[0].colOptions.fk_parent_column_id, - fk_mm_child_column_id: belongsToCols[1].colOptions.fk_child_column_id, - fk_mm_parent_column_id: - belongsToCols[0].colOptions.fk_child_column_id, - type: RelationTypes.MANY_TO_MANY, - uidt: UITypes.LinkToAnotherRecord, - }); - } - - await Model.markAsMmTable(assocModel.id, true); - - // mark has many relation associated with mm as system field in both table - for (const btCol of [belongsToCols[0], belongsToCols[1]]) { - const colOpt = await btCol.colOptions; - const model = await colOpt.getRelatedTable(); - - for (const col of await model.getColumns()) { - if (col.uidt !== UITypes.LinkToAnotherRecord) continue; - - const colOpt1 = await col.getColOptions(); - if (!colOpt1 || colOpt1.type !== RelationTypes.HAS_MANY) continue; - - if ( - colOpt1.fk_child_column_id !== colOpt.fk_child_column_id || - colOpt1.fk_parent_column_id !== colOpt.fk_parent_column_id - ) - continue; - - await Column.markAsSystemField(col.id); - break; - } - } - } else { - if (assocModel.mm) await Model.markAsMmTable(assocModel.id, false); - } - } -} - const router = Router(); router.get( '/api/v1/db/meta/projects/:projectId/meta-diff', diff --git a/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts b/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts index b4a7098da0..1f71c96bde 100644 --- a/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts +++ b/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts @@ -11,7 +11,7 @@ import getTableNameAlias, { import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; import getColumnUiType from '../../helpers/getColumnUiType'; import mapDefaultDisplayValue from '../../helpers/mapDefaultDisplayValue'; -import { extractAndGenerateManyToManyRelations } from '../../../controllers/metaDiffController'; +import { extractAndGenerateManyToManyRelations } from '../../../services/metaDiffService'; import { ModelTypes, UITypes, ViewTypes } from 'nocodb-sdk'; import { IGNORE_TABLES } from '../../../utils/common/BaseApiBuilder'; diff --git a/packages/nocodb/src/lib/models/Model.ts b/packages/nocodb/src/lib/models/Model.ts index 834e8025e4..eb32a3cc7b 100644 --- a/packages/nocodb/src/lib/models/Model.ts +++ b/packages/nocodb/src/lib/models/Model.ts @@ -1,9 +1,9 @@ -import Noco from '../Noco'; -import { parseMetaProp } from '../utils/modelUtils'; -import Column from './Column'; -import NocoCache from '../cache/NocoCache'; -import { XKnex } from '../db/sql-data-mapper'; -import { BaseModelSqlv2 } from '../db/sql-data-mapper/lib/sql/BaseModelSqlv2'; +import Noco from '../Noco' +import { parseMetaProp } from '../utils/modelUtils' +import Column from './Column' +import NocoCache from '../cache/NocoCache' +import { XKnex } from '../db/sql-data-mapper' +import { BaseModelSqlv2 } from '../db/sql-data-mapper/lib/sql/BaseModelSqlv2' import { isVirtualCol, ModelTypes, @@ -12,52 +12,52 @@ import { TableType, UITypes, ViewTypes, -} from 'nocodb-sdk'; +} from 'nocodb-sdk' import { CacheDelDirection, CacheGetType, CacheScope, MetaTable, -} from '../utils/globals'; -import View from './View'; -import { NcError } from '../meta/helpers/catchError'; -import Audit from './Audit'; -import { sanitize } from '../db/sql-data-mapper/lib/sql/helpers/sanitize'; -import { extractProps } from '../meta/helpers/extractProps'; +} from '../utils/globals' +import View from './View' +import { NcError } from '../meta/helpers/catchError' +import Audit from './Audit' +import { sanitize } from '../db/sql-data-mapper/lib/sql/helpers/sanitize' +import { extractProps } from '../meta/helpers/extractProps' export default class Model implements TableType { - copy_enabled: BoolType; - created_at: Date | number | string; - base_id: 'db' | string; - deleted: BoolType; - enabled: BoolType; - export_enabled: BoolType; - id: string; - order: number; - parent_id: string; - password: string; - pin: BoolType; - project_id: string; - schema: any; - show_all_fields: boolean; - tags: string; - type: ModelTypes; - updated_at: Date | number | string; - - table_name: string; - title: string; - - mm: BoolType; - - uuid: string; - - columns?: Column[]; - columnsById?: { [id: string]: Column }; - views?: View[]; - meta?: Record | string; + copy_enabled: BoolType + created_at: Date | number | string + base_id: 'db' | string + deleted: BoolType + enabled: BoolType + export_enabled: BoolType + id: string + order: number + parent_id: string + password: string + pin: BoolType + project_id: string + schema: any + show_all_fields: boolean + tags: string + type: ModelTypes + updated_at: Date | number | string + + table_name: string + title: string + + mm: BoolType + + uuid: string + + columns?: Column[] + columnsById?: { [id: string]: Column } + views?: View[] + meta?: Record | string constructor(data: Partial) { - Object.assign(this, data); + Object.assign(this, data) } public async getColumns(ncMeta = Noco.ncMeta): Promise { @@ -65,34 +65,34 @@ export default class Model implements TableType { { fk_model_id: this.id, }, - ncMeta - ); - return this.columns; + ncMeta, + ) + return this.columns } // @ts-ignore public async getViews(force = false, ncMeta = Noco.ncMeta): Promise { - this.views = await View.listWithInfo(this.id, ncMeta); - return this.views; + this.views = await View.listWithInfo(this.id, ncMeta) + return this.views } public get primaryKey(): Column { - if (!this.columns) return null; - return this.columns?.find((c) => c.pk); + if (!this.columns) return null + return this.columns?.find((c) => c.pk) } public get primaryKeys(): Column[] { - if (!this.columns) return null; - return this.columns?.filter((c) => c.pk); + if (!this.columns) return null + return this.columns?.filter((c) => c.pk) } public get displayValue(): Column { - if (!this.columns) return null; - const pCol = this.columns?.find((c) => c.pv); - if (pCol) return pCol; - const pkIndex = this.columns.indexOf(this.primaryKey); - if (pkIndex < this.columns.length - 1) return this.columns[pkIndex + 1]; - return this.columns[0]; + if (!this.columns) return null + const pCol = this.columns?.find((c) => c.pv) + if (pCol) return pCol + const pkIndex = this.columns.indexOf(this.primaryKey) + if (pkIndex < this.columns.length - 1) return this.columns[pkIndex + 1] + return this.columns[0] } public static async insert( @@ -102,8 +102,9 @@ export default class Model implements TableType { mm?: BoolType; created_at?: any; updated_at?: any; + type?: ModelTypes; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { const insertObj = extractProps(model, [ 'table_name', @@ -114,9 +115,9 @@ export default class Model implements TableType { 'created_at', 'updated_at', 'id', - ]); + ]) - insertObj.mm = !!insertObj.mm; + insertObj.mm = !!insertObj.mm if (!insertObj.order) { insertObj.order = await ncMeta.metaGetNextOrder( @@ -124,26 +125,26 @@ export default class Model implements TableType { { project_id: projectId, base_id: baseId, - } - ); + }, + ) } if (!insertObj.type) { - insertObj.type = ModelTypes.TABLE; + insertObj.type = ModelTypes.TABLE } const { id } = await ncMeta.metaInsert2( projectId, baseId, MetaTable.MODELS, - insertObj - ); + insertObj, + ) await NocoCache.appendToList( CacheScope.MODEL, [projectId], - `${CacheScope.MODEL}:${id}` - ); + `${CacheScope.MODEL}:${id}`, + ) const view = await View.insert( { @@ -154,14 +155,14 @@ export default class Model implements TableType { created_at: model.created_at, updated_at: model.updated_at, }, - ncMeta - ); + ncMeta, + ) for (const column of model?.columns || []) { - await Column.insert({ ...column, fk_model_id: id, view } as any, ncMeta); + await Column.insert({ ...column, fk_model_id: id, view } as any, ncMeta) } - return this.getWithInfo({ id }, ncMeta); + return this.getWithInfo({ id }, ncMeta) } public static async list( @@ -172,13 +173,13 @@ export default class Model implements TableType { project_id: string; base_id: string; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ): Promise { - let modelList = []; + let modelList = [] if (base_id) { - await NocoCache.getList(CacheScope.MODEL, [project_id, base_id]); + await NocoCache.getList(CacheScope.MODEL, [project_id, base_id]) } else { - await NocoCache.getList(CacheScope.MODEL, [project_id]); + await NocoCache.getList(CacheScope.MODEL, [project_id]) } if (!modelList.length) { modelList = await ncMeta.metaList2( @@ -189,30 +190,30 @@ export default class Model implements TableType { orderBy: { order: 'asc', }, - } - ); + }, + ) // parse meta of each model for (const model of modelList) { - model.meta = parseMetaProp(model); + model.meta = parseMetaProp(model) } if (base_id) { await NocoCache.setList( CacheScope.MODEL, [project_id, base_id], - modelList - ); + modelList, + ) } else { - await NocoCache.setList(CacheScope.MODEL, [project_id], modelList); + await NocoCache.setList(CacheScope.MODEL, [project_id], modelList) } } modelList.sort( (a, b) => (a.order != null ? a.order : Infinity) - - (b.order != null ? b.order : Infinity) - ); - return modelList.map((m) => new Model(m)); + (b.order != null ? b.order : Infinity), + ) + return modelList.map((m) => new Model(m)) } public static async listWithInfo( @@ -223,33 +224,33 @@ export default class Model implements TableType { project_id: string; db_alias: string; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ): Promise { let modelList = await NocoCache.getList(CacheScope.MODEL, [ project_id, db_alias, - ]); + ]) if (!modelList.length) { modelList = await ncMeta.metaList2( project_id, db_alias, - MetaTable.MODELS - ); + MetaTable.MODELS, + ) // parse meta of each model for (const model of modelList) { - model.meta = parseMetaProp(model); + model.meta = parseMetaProp(model) } - await NocoCache.setList(CacheScope.MODEL, [project_id], modelList); + await NocoCache.setList(CacheScope.MODEL, [project_id], modelList) } - return modelList.map((m) => new Model(m)); + return modelList.map((m) => new Model(m)) } public static async clear({ id }: { id: string }): Promise { - await NocoCache.delAll(CacheScope.MODEL, `*${id}*`); - await Column.clearList({ fk_model_id: id }); + await NocoCache.delAll(CacheScope.MODEL, `*${id}*`) + await Column.clearList({ fk_model_id: id }) } public static async get(id: string, ncMeta = Noco.ncMeta): Promise { @@ -257,47 +258,47 @@ export default class Model implements TableType { id && (await NocoCache.get( `${CacheScope.MODEL}:${id}`, - CacheGetType.TYPE_OBJECT - )); + CacheGetType.TYPE_OBJECT, + )) if (!modelData) { - modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, id); + modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, id) if (modelData) { - modelData.meta = parseMetaProp(modelData); - await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData); + modelData.meta = parseMetaProp(modelData) + await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData) } } - return modelData && new Model(modelData); + return modelData && new Model(modelData) } public static async getByIdOrName( args: | { - project_id: string; - base_id: string; - table_name: string; - } + project_id: string; + base_id: string; + table_name: string; + } | { - id?: string; - }, - ncMeta = Noco.ncMeta + id?: string; + }, + ncMeta = Noco.ncMeta, ): Promise { - const k = 'id' in args ? args?.id : args; + const k = 'id' in args ? args?.id : args let modelData = k && (await NocoCache.get( `${CacheScope.MODEL}:${k}`, - CacheGetType.TYPE_OBJECT - )); + CacheGetType.TYPE_OBJECT, + )) if (!modelData) { - modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, k); - modelData.meta = parseMetaProp(modelData); + modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, k) + modelData.meta = parseMetaProp(modelData) } if (modelData) { - await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData); - return new Model(modelData); + await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData) + return new Model(modelData) } - return null; + return null } public static async getWithInfo( @@ -308,14 +309,14 @@ export default class Model implements TableType { table_name?: string; id?: string; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ): Promise { let modelData = id && (await NocoCache.get( `${CacheScope.MODEL}:${id}`, - CacheGetType.TYPE_OBJECT - )); + CacheGetType.TYPE_OBJECT, + )) if (!modelData) { modelData = await ncMeta.metaGet2( null, @@ -323,23 +324,23 @@ export default class Model implements TableType { MetaTable.MODELS, id || { table_name, - } - ); - modelData.meta = parseMetaProp(modelData); - await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData); + }, + ) + modelData.meta = parseMetaProp(modelData) + await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData) // modelData.filters = await Filter.getFilterObject({ // viewId: modelData.id // }); // modelData.sorts = await Sort.list({ modelId: modelData.id }); } if (modelData) { - const m = new Model(modelData); - const columns = await m.getColumns(ncMeta); - await m.getViews(false, ncMeta); - m.columnsById = columns.reduce((agg, c) => ({ ...agg, [c.id]: c }), {}); - return m; + const m = new Model(modelData) + const columns = await m.getColumns(ncMeta) + await m.getViews(false, ncMeta) + m.columnsById = columns.reduce((agg, c) => ({ ...agg, [c.id]: c }), {}) + return m } - return null; + return null } public static async getBaseModelSQL( @@ -349,60 +350,60 @@ export default class Model implements TableType { dbDriver: XKnex; model?: Model; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ): Promise { - const model = args?.model || (await this.get(args.id, ncMeta)); + const model = args?.model || (await this.get(args.id, ncMeta)) return new BaseModelSqlv2({ dbDriver: args.dbDriver, viewId: args.viewId, model, - }); + }) } async delete(ncMeta = Noco.ncMeta, force = false): Promise { - await Audit.deleteRowComments(this.id); + await Audit.deleteRowComments(this.id) for (const view of await this.getViews(true)) { - await view.delete(); + await view.delete() } for (const col of await this.getColumns(ncMeta)) { - let colOptionTableName = null; - let cacheScopeName = null; + let colOptionTableName = null + let cacheScopeName = null switch (col.uidt) { case UITypes.Rollup: - colOptionTableName = MetaTable.COL_ROLLUP; - cacheScopeName = CacheScope.COL_ROLLUP; - break; + colOptionTableName = MetaTable.COL_ROLLUP + cacheScopeName = CacheScope.COL_ROLLUP + break case UITypes.Lookup: - colOptionTableName = MetaTable.COL_LOOKUP; - cacheScopeName = CacheScope.COL_LOOKUP; - break; + colOptionTableName = MetaTable.COL_LOOKUP + cacheScopeName = CacheScope.COL_LOOKUP + break case UITypes.ForeignKey: case UITypes.LinkToAnotherRecord: - colOptionTableName = MetaTable.COL_RELATIONS; - cacheScopeName = CacheScope.COL_RELATION; - break; + colOptionTableName = MetaTable.COL_RELATIONS + cacheScopeName = CacheScope.COL_RELATION + break case UITypes.MultiSelect: case UITypes.SingleSelect: - colOptionTableName = MetaTable.COL_SELECT_OPTIONS; - cacheScopeName = CacheScope.COL_SELECT_OPTION; - break; + colOptionTableName = MetaTable.COL_SELECT_OPTIONS + cacheScopeName = CacheScope.COL_SELECT_OPTION + break case UITypes.Formula: - colOptionTableName = MetaTable.COL_FORMULA; - cacheScopeName = CacheScope.COL_FORMULA; - break; + colOptionTableName = MetaTable.COL_FORMULA + cacheScopeName = CacheScope.COL_FORMULA + break } if (colOptionTableName && cacheScopeName) { await ncMeta.metaDelete(null, null, colOptionTableName, { fk_column_id: col.id, - }); + }) await NocoCache.deepDel( cacheScopeName, `${cacheScopeName}:${col.id}`, - CacheDelDirection.CHILD_TO_PARENT - ); + CacheDelDirection.CHILD_TO_PARENT, + ) } } @@ -415,82 +416,82 @@ export default class Model implements TableType { condition: { fk_related_model_id: this.id, }, - } - ); + }, + ) for (const col of leftOverColumns) { await NocoCache.deepDel( CacheScope.COL_RELATION, `${CacheScope.COL_RELATION}:${col.fk_column_id}`, - CacheDelDirection.CHILD_TO_PARENT - ); + CacheDelDirection.CHILD_TO_PARENT, + ) } await ncMeta.metaDelete(null, null, MetaTable.COL_RELATIONS, { fk_related_model_id: this.id, - }); + }) } await NocoCache.deepDel( CacheScope.COLUMN, `${CacheScope.COLUMN}:${this.id}`, - CacheDelDirection.CHILD_TO_PARENT - ); + CacheDelDirection.CHILD_TO_PARENT, + ) await ncMeta.metaDelete(null, null, MetaTable.COLUMNS, { fk_model_id: this.id, - }); + }) await NocoCache.deepDel( CacheScope.MODEL, `${CacheScope.MODEL}:${this.id}`, - CacheDelDirection.CHILD_TO_PARENT - ); - await ncMeta.metaDelete(null, null, MetaTable.MODELS, this.id); + CacheDelDirection.CHILD_TO_PARENT, + ) + await ncMeta.metaDelete(null, null, MetaTable.MODELS, this.id) - await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.id}`); - await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.title}`); - return true; + await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.id}`) + await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.title}`) + return true } async mapAliasToColumn(data) { - const insertObj = {}; + const insertObj = {} for (const col of await this.getColumns()) { - if (isVirtualCol(col)) continue; + if (isVirtualCol(col)) continue let val = data?.[col.column_name] !== undefined ? data?.[col.column_name] - : data?.[col.title]; + : data?.[col.title] if (val !== undefined) { if (col.uidt === UITypes.Attachment && typeof val !== 'string') { - val = JSON.stringify(val); + val = JSON.stringify(val) } - insertObj[sanitize(col.column_name)] = val; + insertObj[sanitize(col.column_name)] = val } } - return insertObj; + return insertObj } static async updateAliasAndTableName( tableId, title: string, table_name: string, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { if (!title) { - NcError.badRequest("Missing 'title' property in body"); + NcError.badRequest('Missing \'title\' property in body') } if (!table_name) { - NcError.badRequest("Missing 'table_name' property in body"); + NcError.badRequest('Missing \'table_name\' property in body') } // get existing cache - const key = `${CacheScope.MODEL}:${tableId}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.MODEL}:${tableId}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) // update alias if (o) { - o.title = title; - o.table_name = table_name; + o.title = title + o.table_name = table_name // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta return await ncMeta.metaUpdate( @@ -501,19 +502,19 @@ export default class Model implements TableType { title, table_name, }, - tableId - ); + tableId, + ) } static async markAsMmTable(tableId, isMm = true, ncMeta = Noco.ncMeta) { // get existing cache - const key = `${CacheScope.MODEL}:${tableId}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.MODEL}:${tableId}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) // update alias if (o) { - o.mm = isMm; + o.mm = isMm // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta return await ncMeta.metaUpdate( @@ -523,40 +524,40 @@ export default class Model implements TableType { { mm: isMm, }, - tableId - ); + tableId, + ) } async getAliasColMapping() { return (await this.getColumns()).reduce((o, c) => { if (c.column_name) { - o[c.title] = c.column_name; + o[c.title] = c.column_name } - return o; - }, {}); + return o + }, {}) } async getColAliasMapping() { return (await this.getColumns()).reduce((o, c) => { if (c.column_name) { - o[c.column_name] = c.title; + o[c.column_name] = c.title } - return o; - }, {}); + return o + }, {}) } static async updateOrder( tableId: string, order: number, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { // get existing cache - const key = `${CacheScope.MODEL}:${tableId}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.MODEL}:${tableId}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (o) { - o.order = order; + o.order = order // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta return await ncMeta.metaUpdate( @@ -566,29 +567,29 @@ export default class Model implements TableType { { order, }, - tableId - ); + tableId, + ) } static async updatePrimaryColumn( tableId: string, columnId: string, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { - const model = await this.getWithInfo({ id: tableId }); - const newPvCol = model.columns.find((c) => c.id === columnId); + const model = await this.getWithInfo({ id: tableId }) + const newPvCol = model.columns.find((c) => c.id === columnId) - if (!newPvCol) NcError.badRequest('Column not found'); + if (!newPvCol) NcError.badRequest('Column not found') // drop existing primary column/s for (const col of model.columns?.filter((c) => c.pv) || []) { // get existing cache - const key = `${CacheScope.COLUMN}:${col.id}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.COLUMN}:${col.id}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (o) { - o.pv = false; + o.pv = false // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta await ncMeta.metaUpdate( @@ -598,17 +599,17 @@ export default class Model implements TableType { { pv: false, }, - col.id - ); + col.id, + ) } // get existing cache - const key = `${CacheScope.COLUMN}:${newPvCol.id}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.COLUMN}:${newPvCol.id}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (o) { - o.pv = true; + o.pv = true // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta await ncMeta.metaUpdate( @@ -618,8 +619,8 @@ export default class Model implements TableType { { pv: true, }, - newPvCol.id - ); + newPvCol.id, + ) const grid_views_with_column = await ncMeta.metaList2( null, @@ -629,26 +630,26 @@ export default class Model implements TableType { condition: { fk_column_id: newPvCol.id, }, - } - ); + }, + ) if (grid_views_with_column.length) { for (const gv of grid_views_with_column) { - await View.fixPVColumnForView(gv.fk_view_id, ncMeta); + await View.fixPVColumnForView(gv.fk_view_id, ncMeta) } } - return true; + return true } static async setAsMm(id: any, ncMeta = Noco.ncMeta) { // get existing cache - const key = `${CacheScope.MODEL}:${id}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.MODEL}:${id}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (o) { - o.mm = true; + o.mm = true // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta await ncMeta.metaUpdate( @@ -658,8 +659,8 @@ export default class Model implements TableType { { mm: true, }, - id - ); + id, + ) } static async getByAliasOrId( @@ -672,69 +673,69 @@ export default class Model implements TableType { base_id?: string; aliasOrId: string; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { const modelId = project_id && aliasOrId && (await NocoCache.get( `${CacheScope.MODEL}:${project_id}:${aliasOrId}`, - CacheGetType.TYPE_OBJECT - )); + CacheGetType.TYPE_OBJECT, + )) if (!modelId) { const model = base_id ? await ncMeta.metaGet2( - null, - null, - MetaTable.MODELS, - { project_id, base_id }, - null, - { - _or: [ - { - id: { - eq: aliasOrId, - }, + null, + null, + MetaTable.MODELS, + { project_id, base_id }, + null, + { + _or: [ + { + id: { + eq: aliasOrId, }, - { - title: { - eq: aliasOrId, - }, + }, + { + title: { + eq: aliasOrId, }, - ], - } - ) + }, + ], + }, + ) : await ncMeta.metaGet2( - null, - null, - MetaTable.MODELS, - { project_id }, - null, - { - _or: [ - { - id: { - eq: aliasOrId, - }, + null, + null, + MetaTable.MODELS, + { project_id }, + null, + { + _or: [ + { + id: { + eq: aliasOrId, }, - { - title: { - eq: aliasOrId, - }, + }, + { + title: { + eq: aliasOrId, }, - ], - } - ); + }, + ], + }, + ) if (model) { await NocoCache.set( `${CacheScope.MODEL}:${project_id}:${aliasOrId}`, - model.id - ); - await NocoCache.set(`${CacheScope.MODEL}:${model.id}`, model); + model.id, + ) + await NocoCache.set(`${CacheScope.MODEL}:${model.id}`, model) } - return model && new Model(model); + return model && new Model(model) } - return modelId && this.get(modelId); + return modelId && this.get(modelId) } static async checkTitleAvailable( @@ -744,7 +745,7 @@ export default class Model implements TableType { base_id, exclude_id, }: { table_name; project_id; base_id; exclude_id? }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { return !(await ncMeta.metaGet2( project_id, @@ -754,8 +755,8 @@ export default class Model implements TableType { table_name, }, null, - exclude_id && { id: { neq: exclude_id } } - )); + exclude_id && { id: { neq: exclude_id } }, + )) } static async checkAliasAvailable( @@ -765,7 +766,7 @@ export default class Model implements TableType { base_id, exclude_id, }: { title; project_id; base_id; exclude_id? }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { return !(await ncMeta.metaGet2( project_id, @@ -775,32 +776,33 @@ export default class Model implements TableType { title, }, null, - exclude_id && { id: { neq: exclude_id } } - )); + exclude_id && { id: { neq: exclude_id } }, + )) } async getAliasColObjMap() { return (await this.getColumns()).reduce( (sortAgg, c) => ({ ...sortAgg, [c.title]: c }), - {} - ); + {}, + ) } // For updating table meta static async updateMeta( tableId: string, meta: string | Record, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { // get existing cache - const key = `${CacheScope.MODEL}:${tableId}`; - const existingCache = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.MODEL}:${tableId}` + const existingCache = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (existingCache) { try { - existingCache.meta = typeof meta === 'string' ? JSON.parse(meta) : meta; + existingCache.meta = typeof meta === 'string' ? JSON.parse(meta) : meta // set cache - await NocoCache.set(key, existingCache); - } catch {} + await NocoCache.set(key, existingCache) + } catch { + } } // set meta return await ncMeta.metaUpdate( @@ -810,7 +812,7 @@ export default class Model implements TableType { { meta: typeof meta === 'object' ? JSON.stringify(meta) : meta, }, - tableId - ); + tableId, + ) } } diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index d4998f493b..614dfd5ee2 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -16,3 +16,5 @@ export * as galleryViewService from './galleryViewService'; export * as kanbanViewService from './kanbanViewService'; export * as gridViewColumnService from './gridViewColumnService'; export * as viewColumnService from './viewColumnService'; +export * as metaDiffService from './metaDiffService'; +export * as mapViewService from './mapViewService'; diff --git a/packages/nocodb/src/lib/services/mapViewService.ts b/packages/nocodb/src/lib/services/mapViewService.ts new file mode 100644 index 0000000000..ba21af1b6c --- /dev/null +++ b/packages/nocodb/src/lib/services/mapViewService.ts @@ -0,0 +1,34 @@ +import { MapType, ViewTypes } from 'nocodb-sdk'; +import View from '../models/View'; +import { Tele } from 'nc-help'; +import MapView from '../models/MapView'; + +export async function mapViewGet(param:{mapViewId: string}) { + return await MapView.get(param.mapViewId) +} + +export async function mapViewCreate(param:{ + tableId: string, + // todo: add MapReq in schema + map: MapType +}) { + Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'map' }); + const view = await View.insert({ + ...param.map, + // todo: sanitize + fk_model_id: param.tableId, + type: ViewTypes.MAP, + }); + return view; +} + +export async function mapViewUpdate(param:{ + mapViewId: string, + // todo: add MapReq in schema + map: MapType + +}) { + Tele.emit('evt', { evt_type: 'view:updated', type: 'map' }); + // todo: type correction + return await MapView.update(param.mapViewId, param.map as any) +} diff --git a/packages/nocodb/src/lib/services/metaDiffService.ts b/packages/nocodb/src/lib/services/metaDiffService.ts new file mode 100644 index 0000000000..0786a0c19a --- /dev/null +++ b/packages/nocodb/src/lib/services/metaDiffService.ts @@ -0,0 +1,1105 @@ +// // Project CRUD + +import { Tele } from 'nc-help'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import { isVirtualCol, ModelTypes, RelationTypes, UITypes } from 'nocodb-sdk'; +import { + Base, + Column, + LinkToAnotherRecordColumn, + Model, + Project, +} from '../models'; +import ModelXcMetaFactory from '../db/sql-mgr/code/models/xc/ModelXcMetaFactory'; +import { getUniqueColumnAliasName } from '../meta/helpers/getUniqueName'; +import NcHelp from '../utils/NcHelp'; +import getTableNameAlias, { + getColumnNameAlias, +} from '../meta/helpers/getTableName'; +import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; +import getColumnUiType from '../meta/helpers/getColumnUiType'; + +export enum MetaDiffType { + TABLE_NEW = 'TABLE_NEW', + TABLE_REMOVE = 'TABLE_REMOVE', + TABLE_COLUMN_ADD = 'TABLE_COLUMN_ADD', + TABLE_COLUMN_TYPE_CHANGE = 'TABLE_COLUMN_TYPE_CHANGE', + TABLE_COLUMN_REMOVE = 'TABLE_COLUMN_REMOVE', + VIEW_NEW = 'VIEW_NEW', + VIEW_REMOVE = 'VIEW_REMOVE', + VIEW_COLUMN_ADD = 'VIEW_COLUMN_ADD', + VIEW_COLUMN_TYPE_CHANGE = 'VIEW_COLUMN_TYPE_CHANGE', + VIEW_COLUMN_REMOVE = 'VIEW_COLUMN_REMOVE', + TABLE_RELATION_ADD = 'TABLE_RELATION_ADD', + TABLE_RELATION_REMOVE = 'TABLE_RELATION_REMOVE', + TABLE_VIRTUAL_M2M_REMOVE = 'TABLE_VIRTUAL_M2M_REMOVE', +} + +const applyChangesPriorityOrder = [ + MetaDiffType.VIEW_COLUMN_REMOVE, + MetaDiffType.TABLE_RELATION_REMOVE, +]; + +type MetaDiff = { + title?: string; + table_name: string; + base_id: string; + type: ModelTypes; + meta?: any; + detectedChanges: Array; +}; + +type MetaDiffChange = { + msg?: string; + // type: MetaDiffType; +} & ( + | { + type: MetaDiffType.TABLE_NEW | MetaDiffType.VIEW_NEW; + tn?: string; + } + | { + type: MetaDiffType.TABLE_REMOVE | MetaDiffType.VIEW_REMOVE; + tn?: string; + model?: Model; + id?: string; + } + | { + type: MetaDiffType.TABLE_COLUMN_ADD | MetaDiffType.VIEW_COLUMN_ADD; + tn?: string; + model?: Model; + id?: string; + cn: string; + } + | { + type: + | MetaDiffType.TABLE_COLUMN_TYPE_CHANGE + | MetaDiffType.VIEW_COLUMN_TYPE_CHANGE + | MetaDiffType.TABLE_COLUMN_REMOVE + | MetaDiffType.VIEW_COLUMN_REMOVE; + tn?: string; + model?: Model; + id?: string; + cn: string; + column: Column; + colId?: string; + } + | { + type: MetaDiffType.TABLE_RELATION_REMOVE; + tn?: string; + rtn?: string; + cn?: string; + rcn?: string; + colId: string; + column: Column; + } + | { + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE; + tn?: string; + rtn?: string; + cn?: string; + rcn?: string; + colId: string; + column: Column; + } + | { + type: MetaDiffType.TABLE_RELATION_ADD; + tn?: string; + rtn?: string; + cn?: string; + rcn?: string; + relationType: RelationTypes; + cstn?: string; + } +); + +async function getMetaDiff( + sqlClient, + project: Project, + base: Base +): Promise> { + const changes: Array = []; + const virtualRelationColumns: Column[] = []; + + // @ts-ignore + const tableList: Array<{ tn: string }> = ( + await sqlClient.tableList() + )?.data?.list?.filter((t) => { + if (project?.prefix && base.is_meta) { + return t.tn?.startsWith(project?.prefix); + } + return true; + }); + + const colListRef = {}; + const oldMetas = await base.getModels(); + // @ts-ignore + const oldTableMetas: Model[] = []; + const oldViewMetas: Model[] = []; + + for (const model of oldMetas) { + if (model.type === ModelTypes.TABLE) oldTableMetas.push(model); + else if (model.type === ModelTypes.VIEW) oldViewMetas.push(model); + } + + // @ts-ignore + const relationList: Array<{ + tn: string; + rtn: string; + cn: string; + rcn: string; + found?: any; + cstn?: string; + }> = (await sqlClient.relationListAll())?.data?.list; + + for (const table of tableList) { + if (table.tn === 'nc_evolutions') continue; + + const oldMetaIdx = oldTableMetas.findIndex( + (m) => m.table_name === table.tn + ); + + // new table + if (oldMetaIdx === -1) { + changes.push({ + table_name: table.tn, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [ + { + type: MetaDiffType.TABLE_NEW, + msg: `New table`, + }, + ], + }); + continue; + } + + const oldMeta = oldTableMetas[oldMetaIdx]; + + oldTableMetas.splice(oldMetaIdx, 1); + + const tableProp: MetaDiff = { + title: oldMeta.title, + meta: oldMeta.meta, + table_name: table.tn, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [], + }; + changes.push(tableProp); + + // check for column change + colListRef[table.tn] = ( + await sqlClient.columnList({ tn: table.tn }) + )?.data?.list; + + await oldMeta.getColumns(); + + for (const column of colListRef[table.tn]) { + const oldColIdx = oldMeta.columns.findIndex( + (c) => c.column_name === column.cn + ); + + // new table + if (oldColIdx === -1) { + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_ADD, + msg: `New column(${column.cn})`, + cn: column.cn, + id: oldMeta.id, + }); + continue; + } + + const [oldCol] = oldMeta.columns.splice(oldColIdx, 1); + + if (oldCol.dt !== column.dt) { + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_TYPE_CHANGE, + msg: `Column type changed(${column.cn})`, + cn: oldCol.column_name, + id: oldMeta.id, + column: oldCol, + }); + } + } + for (const column of oldMeta.columns) { + if ( + [ + UITypes.LinkToAnotherRecord, + UITypes.Rollup, + UITypes.Lookup, + UITypes.Formula, + ].includes(column.uidt) + ) { + if (column.uidt === UITypes.LinkToAnotherRecord) { + virtualRelationColumns.push(column); + } + + continue; + } + + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_REMOVE, + msg: `Column removed(${column.column_name})`, + cn: column.column_name, + id: oldMeta.id, + column: column, + colId: column.id, + }); + } + } + + for (const model of oldTableMetas) { + changes.push({ + table_name: model.table_name, + meta: model.meta, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [ + { + type: MetaDiffType.TABLE_REMOVE, + msg: `Table removed`, + tn: model.table_name, + id: model.id, + model, + }, + ], + }); + } + + for (const relationCol of virtualRelationColumns) { + const colOpt = await relationCol.getColOptions(); + const parentCol = await colOpt.getParentColumn(); + const childCol = await colOpt.getChildColumn(); + const parentModel = await parentCol.getModel(); + const childModel = await childCol.getModel(); + + // many to many relation + if (colOpt.type === RelationTypes.MANY_TO_MANY) { + const m2mModel = await colOpt.getMMModel(); + + const relatedTable = tableList.find( + (t) => t.tn === parentModel.table_name + ); + const m2mTable = tableList.find((t) => t.tn === m2mModel.table_name); + + if (!relatedTable) { + changes + .find((t) => t.table_name === childModel.table_name) + .detectedChanges.push({ + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, + msg: `Many to many removed(${relatedTable.tn} removed)`, + colId: relationCol.id, + column: relationCol, + }); + continue; + } + if (!m2mTable) { + changes + .find((t) => t.table_name === childModel.table_name) + .detectedChanges.push({ + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, + msg: `Many to many removed(${m2mModel.table_name} removed)`, + colId: relationCol.id, + column: relationCol, + }); + continue; + } + + // verify columns + + const cColumns = (colListRef[childModel.table_name] = + colListRef[childModel.table_name] || + (await sqlClient.columnList({ tn: childModel.table_name }))?.data + ?.list); + + const pColumns = (colListRef[parentModel.table_name] = + colListRef[parentModel.table_name] || + (await sqlClient.columnList({ tn: parentModel.table_name }))?.data + ?.list); + + const vColumns = (colListRef[m2mTable.tn] = + colListRef[m2mTable.tn] || + (await sqlClient.columnList({ tn: m2mTable.tn }))?.data?.list); + + const m2mChildCol = await colOpt.getMMChildColumn(); + const m2mParentCol = await colOpt.getMMParentColumn(); + + if ( + pColumns.every((c) => c.cn !== parentCol.column_name) || + cColumns.every((c) => c.cn !== childCol.column_name) || + vColumns.every((c) => c.cn !== m2mChildCol.column_name) || + vColumns.every((c) => c.cn !== m2mParentCol.column_name) + ) { + changes + .find((t) => t.table_name === childModel.table_name) + .detectedChanges.push({ + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, + msg: `Many to many removed(One of the relation column removed)`, + colId: relationCol.id, + column: relationCol, + }); + } + + continue; + } + + if (relationCol.colOptions.virtual) continue; + + const dbRelation = relationList.find( + (r) => + r.cn === childCol.column_name && + r.tn === childModel.table_name && + r.rcn === parentCol.column_name && + r.rtn === parentModel.table_name + ); + + if (dbRelation) { + dbRelation.found = dbRelation.found || {}; + + if (dbRelation.found[colOpt.type]) { + // todo: handle duplicate + } else { + dbRelation.found[colOpt.type] = true; + } + } else { + changes + .find( + (t) => + t.table_name === + (colOpt.type === RelationTypes.BELONGS_TO + ? childModel.table_name + : parentModel.table_name) + ) + .detectedChanges.push({ + type: MetaDiffType.TABLE_RELATION_REMOVE, + tn: childModel.table_name, + rtn: parentModel.table_name, + cn: childCol.column_name, + rcn: parentCol.column_name, + msg: `Relation removed`, + colId: relationCol.id, + column: relationCol, + }); + } + } + + for (const relation of relationList) { + if (!relation?.found?.[RelationTypes.BELONGS_TO]) { + changes + .find((t) => t.table_name === relation.tn) + ?.detectedChanges.push({ + type: MetaDiffType.TABLE_RELATION_ADD, + tn: relation.tn, + rtn: relation.rtn, + cn: relation.cn, + rcn: relation.rcn, + msg: `New relation added`, + relationType: RelationTypes.BELONGS_TO, + cstn: relation.cstn, + }); + } + if (!relation?.found?.[RelationTypes.HAS_MANY]) { + changes + .find((t) => t.table_name === relation.rtn) + ?.detectedChanges.push({ + type: MetaDiffType.TABLE_RELATION_ADD, + tn: relation.tn, + rtn: relation.rtn, + cn: relation.cn, + rcn: relation.rcn, + msg: `New relation added`, + relationType: RelationTypes.HAS_MANY, + }); + } + } + + // views + // @ts-ignore + const viewList: Array<{ + view_name: string; + tn: string; + type: 'view'; + }> = (await sqlClient.viewList())?.data?.list + ?.map((v) => { + v.type = 'view'; + v.tn = v.view_name; + return v; + }) + .filter((t) => { + if (project?.prefix && base.is_meta) { + return t.tn?.startsWith(project?.prefix); + } + return true; + }); // @ts-ignore + + for (const view of viewList) { + const oldMetaIdx = oldViewMetas.findIndex((m) => m.table_name === view.tn); + + // new table + if (oldMetaIdx === -1) { + changes.push({ + table_name: view.tn, + base_id: base.id, + type: ModelTypes.VIEW, + detectedChanges: [ + { + type: MetaDiffType.VIEW_NEW, + msg: `New view`, + }, + ], + }); + continue; + } + + const oldMeta = oldViewMetas[oldMetaIdx]; + + oldViewMetas.splice(oldMetaIdx, 1); + + const tableProp: MetaDiff = { + title: oldMeta.title, + meta: oldMeta.meta, + table_name: view.tn, + base_id: base.id, + type: ModelTypes.VIEW, + detectedChanges: [], + }; + changes.push(tableProp); + + // check for column change + colListRef[view.tn] = ( + await sqlClient.columnList({ tn: view.tn }) + )?.data?.list; + + await oldMeta.getColumns(); + + for (const column of colListRef[view.tn]) { + const oldColIdx = oldMeta.columns.findIndex( + (c) => c.column_name === column.cn + ); + + // new table + if (oldColIdx === -1) { + tableProp.detectedChanges.push({ + type: MetaDiffType.VIEW_COLUMN_ADD, + msg: `New column(${column.cn})`, + cn: column.cn, + id: oldMeta.id, + }); + continue; + } + + const [oldCol] = oldMeta.columns.splice(oldColIdx, 1); + + if (oldCol.dt !== column.dt) { + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_TYPE_CHANGE, + msg: `Column type changed(${column.cn})`, + cn: oldCol.column_name, + id: oldMeta.id, + column: oldCol, + }); + } + } + for (const column of oldMeta.columns) { + if ( + [ + UITypes.LinkToAnotherRecord, + UITypes.Rollup, + UITypes.Lookup, + UITypes.Formula, + ].includes(column.uidt) + ) { + continue; + } + + tableProp.detectedChanges.push({ + type: MetaDiffType.VIEW_COLUMN_REMOVE, + msg: `Column removed(${column.column_name})`, + cn: column.column_name, + id: oldMeta.id, + column: column, + colId: column.id, + }); + } + } + + for (const model of oldViewMetas) { + changes.push({ + table_name: model.table_name, + meta: model.meta, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [ + { + type: MetaDiffType.VIEW_REMOVE, + msg: `Table removed`, + tn: model.table_name, + id: model.id, + model, + }, + ], + }); + } + + return changes; +} + +export async function metaDiff(param: { projectId: string }) { + const project = await Project.getWithInfo(param.projectId); + let changes = []; + for (const base of project.bases) { + try { + // @ts-ignore + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + changes = changes.concat(await getMetaDiff(sqlClient, project, base)); + } catch (e) { + console.log(e); + } + } + + return changes; +} + +export async function baseMetaDiff(param: { + projectId: string; + baseId: string; +}) { + const project = await Project.getWithInfo(param.projectId); + const base = await Base.get(param.baseId); + let changes = []; + + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + changes = await getMetaDiff(sqlClient, project, base); + + return changes; +} + +export async function metaDiffSync(param: { + projectId: string; +}) { + const project = await Project.getWithInfo(param.projectId); + for (const base of project.bases) { + const virtualColumnInsert: Array<() => Promise> = []; + + // @ts-ignore + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + const changes = await getMetaDiff(sqlClient, project, base); + + /* Get all relations */ + // const relations = (await sqlClient.relationListAll())?.data?.list; + + for (const { table_name, detectedChanges } of changes) { + // reorder changes to apply relation remove changes + // before column remove to avoid foreign key constraint error + detectedChanges.sort((a, b) => { + return ( + applyChangesPriorityOrder.indexOf(b.type) - + applyChangesPriorityOrder.indexOf(a.type) + ); + }); + + for (const change of detectedChanges) { + switch (change.type) { + case MetaDiffType.TABLE_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias( + table_name, + base.is_meta ? project.prefix : '', + base + ), + type: ModelTypes.TABLE, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.VIEW_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias(table_name, project.prefix, base), + type: ModelTypes.VIEW, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.TABLE_REMOVE: + case MetaDiffType.VIEW_REMOVE: + { + await change.model.delete(); + } + break; + case MetaDiffType.TABLE_COLUMN_ADD: + case MetaDiffType.VIEW_COLUMN_ADD: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + column.uidt = getColumnUiType(base, column); + //todo: inflection + column.title = getColumnNameAlias(column.cn, base); + await Column.insert({ fk_model_id: change.id, ...column }); + } + // update old + // populateParams.tableNames.push({ tn }); + // populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn); + + break; + case MetaDiffType.TABLE_COLUMN_TYPE_CHANGE: + case MetaDiffType.VIEW_COLUMN_TYPE_CHANGE: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + const metaFact = ModelXcMetaFactory.create( + { client: base.type }, + {} + ); + column.uidt = metaFact.getUIDataType(column); + column.title = change.column.title; + await Column.update(change.column.id, column); + } + break; + case MetaDiffType.TABLE_COLUMN_REMOVE: + case MetaDiffType.VIEW_COLUMN_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_REMOVE: + case MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_ADD: + { + virtualColumnInsert.push(async () => { + const parentModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.rtn, + }); + const childModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.tn, + }); + const parentCol = await parentModel + .getColumns() + .then((cols) => + cols.find((c) => c.column_name === change.rcn) + ); + const childCol = await childModel + .getColumns() + .then((cols) => + cols.find((c) => c.column_name === change.cn) + ); + + await Column.update(childCol.id, { + ...childCol, + uidt: UITypes.ForeignKey, + system: true, + }); + + if (change.relationType === RelationTypes.BELONGS_TO) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${parentModel.title || parentModel.table_name}` + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: childModel.id, + fk_related_model_id: parentModel.id, + type: RelationTypes.BELONGS_TO, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + fk_index_name: change.cstn, + }); + } else if (change.relationType === RelationTypes.HAS_MANY) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${childModel.title || childModel.table_name} List` + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: parentModel.id, + fk_related_model_id: childModel.id, + type: RelationTypes.HAS_MANY, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + fk_index_name: change.cstn, + }); + } + }); + } + break; + } + } + } + + await NcHelp.executeOperations(virtualColumnInsert, base.type); + + // populate m2m relations + await extractAndGenerateManyToManyRelations(await base.getModels()); + } + + Tele.emit('evt', { evt_type: 'metaDiff:synced' }); + + return true; +} + +export async function baseMetaDiffSync(param: { + projectId: string; + baseId: string; +}) { + const project = await Project.getWithInfo(param.projectId); + const base = await Base.get(param.baseId); + + const virtualColumnInsert: Array<() => Promise> = []; + + // @ts-ignore + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + const changes = await getMetaDiff(sqlClient, project, base); + + /* Get all relations */ + // const relations = (await sqlClient.relationListAll())?.data?.list; + + for (const { table_name, detectedChanges } of changes) { + for (const change of detectedChanges) { + switch (change.type) { + case MetaDiffType.TABLE_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias( + table_name, + base.is_meta ? project.prefix : '', + base + ), + type: ModelTypes.TABLE, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.VIEW_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias(table_name, project.prefix, base), + type: ModelTypes.VIEW, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.TABLE_REMOVE: + case MetaDiffType.VIEW_REMOVE: + { + await change.model.delete(); + } + break; + case MetaDiffType.TABLE_COLUMN_ADD: + case MetaDiffType.VIEW_COLUMN_ADD: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + column.uidt = getColumnUiType(base, column); + //todo: inflection + column.title = getColumnNameAlias(column.cn, base); + await Column.insert({ fk_model_id: change.id, ...column }); + } + // update old + // populateParams.tableNames.push({ tn }); + // populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn); + + break; + case MetaDiffType.TABLE_COLUMN_TYPE_CHANGE: + case MetaDiffType.VIEW_COLUMN_TYPE_CHANGE: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + const metaFact = ModelXcMetaFactory.create( + { client: base.type }, + {} + ); + column.uidt = metaFact.getUIDataType(column); + column.title = change.column.title; + await Column.update(change.column.id, column); + } + break; + case MetaDiffType.TABLE_COLUMN_REMOVE: + case MetaDiffType.VIEW_COLUMN_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_REMOVE: + case MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_ADD: + { + virtualColumnInsert.push(async () => { + const parentModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.rtn, + }); + const childModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.tn, + }); + const parentCol = await parentModel + .getColumns() + .then((cols) => cols.find((c) => c.column_name === change.rcn)); + const childCol = await childModel + .getColumns() + .then((cols) => cols.find((c) => c.column_name === change.cn)); + + await Column.update(childCol.id, { + ...childCol, + uidt: UITypes.ForeignKey, + system: true, + }); + + if (change.relationType === RelationTypes.BELONGS_TO) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${parentModel.title || parentModel.table_name}` + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: childModel.id, + fk_related_model_id: parentModel.id, + type: RelationTypes.BELONGS_TO, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + }); + } else if (change.relationType === RelationTypes.HAS_MANY) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${childModel.title || childModel.table_name} List` + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: parentModel.id, + fk_related_model_id: childModel.id, + type: RelationTypes.HAS_MANY, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + }); + } + }); + } + break; + } + } + } + + await NcHelp.executeOperations(virtualColumnInsert, base.type); + + // populate m2m relations + await extractAndGenerateManyToManyRelations(await base.getModels()); + + Tele.emit('evt', { evt_type: 'baseMetaDiff:synced' }); + + return true; +} + +async function isMMRelationExist( + model: Model, + assocModel: Model, + belongsToCol: Column +) { + let isExist = false; + const colChildOpt = + await belongsToCol.getColOptions(); + for (const col of await model.getColumns()) { + if (col.uidt === UITypes.LinkToAnotherRecord) { + const colOpt = await col.getColOptions(); + if ( + colOpt && + colOpt.type === RelationTypes.MANY_TO_MANY && + colOpt.fk_mm_model_id === assocModel.id && + colOpt.fk_child_column_id === colChildOpt.fk_parent_column_id && + colOpt.fk_mm_child_column_id === colChildOpt.fk_child_column_id + ) { + isExist = true; + break; + } + } + } + return isExist; +} + +// @ts-ignore +export async function extractAndGenerateManyToManyRelations( + modelsArr: Array +) { + for (const assocModel of modelsArr) { + await assocModel.getColumns(); + // check if table is a Bridge table(or Associative Table) by checking + // number of foreign keys and columns + + const normalColumns = assocModel.columns.filter((c) => !isVirtualCol(c)); + const belongsToCols: Column[] = []; + for (const col of assocModel.columns) { + if (col.uidt == UITypes.LinkToAnotherRecord) { + const colOpt = await col.getColOptions(); + if (colOpt?.type === RelationTypes.BELONGS_TO) belongsToCols.push(col); + } + } + + // todo: impl better method to identify m2m relation + if (belongsToCols?.length === 2 && normalColumns.length < 5) { + const modelA = await belongsToCols[0].colOptions.getRelatedTable(); + const modelB = await belongsToCols[1].colOptions.getRelatedTable(); + + await modelA.getColumns(); + await modelB.getColumns(); + + // check tableA already have the relation or not + const isRelationAvailInA = await isMMRelationExist( + modelA, + assocModel, + belongsToCols[0] + ); + const isRelationAvailInB = await isMMRelationExist( + modelB, + assocModel, + belongsToCols[1] + ); + + if (!isRelationAvailInA) { + await Column.insert({ + title: getUniqueColumnAliasName( + modelA.columns, + `${modelB.title} List` + ), + fk_model_id: modelA.id, + fk_related_model_id: modelB.id, + fk_mm_model_id: assocModel.id, + fk_child_column_id: belongsToCols[0].colOptions.fk_parent_column_id, + fk_parent_column_id: belongsToCols[1].colOptions.fk_parent_column_id, + fk_mm_child_column_id: belongsToCols[0].colOptions.fk_child_column_id, + fk_mm_parent_column_id: + belongsToCols[1].colOptions.fk_child_column_id, + type: RelationTypes.MANY_TO_MANY, + uidt: UITypes.LinkToAnotherRecord, + }); + } + if (!isRelationAvailInB) { + await Column.insert({ + title: getUniqueColumnAliasName( + modelB.columns, + `${modelA.title} List` + ), + fk_model_id: modelB.id, + fk_related_model_id: modelA.id, + fk_mm_model_id: assocModel.id, + fk_child_column_id: belongsToCols[1].colOptions.fk_parent_column_id, + fk_parent_column_id: belongsToCols[0].colOptions.fk_parent_column_id, + fk_mm_child_column_id: belongsToCols[1].colOptions.fk_child_column_id, + fk_mm_parent_column_id: + belongsToCols[0].colOptions.fk_child_column_id, + type: RelationTypes.MANY_TO_MANY, + uidt: UITypes.LinkToAnotherRecord, + }); + } + + await Model.markAsMmTable(assocModel.id, true); + + // mark has many relation associated with mm as system field in both table + for (const btCol of [belongsToCols[0], belongsToCols[1]]) { + const colOpt = await btCol.colOptions; + const model = await colOpt.getRelatedTable(); + + for (const col of await model.getColumns()) { + if (col.uidt !== UITypes.LinkToAnotherRecord) continue; + + const colOpt1 = await col.getColOptions(); + if (!colOpt1 || colOpt1.type !== RelationTypes.HAS_MANY) continue; + + if ( + colOpt1.fk_child_column_id !== colOpt.fk_child_column_id || + colOpt1.fk_parent_column_id !== colOpt.fk_parent_column_id + ) + continue; + + await Column.markAsSystemField(col.id); + break; + } + } + } else { + if (assocModel.mm) await Model.markAsMmTable(assocModel.id, false); + } + } +} From 50f8266167375b57190299d3f90454bc2fdce9bb Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Mar 2023 11:36:18 +0530 Subject: [PATCH 09/64] refcator: handler to service-controller - modelvisibility and verify view belongs to the project Signed-off-by: Pranav C --- packages/nocodb-sdk/src/lib/Api.ts | 1 + .../controllers/modelVisibilityController.ts | 115 ++---------------- packages/nocodb/src/lib/meta/api/tableApis.ts | 0 packages/nocodb/src/lib/services/index.ts | 1 + .../lib/services/modelVisibilityService.ts | 97 +++++++++++++++ .../nocodb/src/lib/services/viewService.ts | 2 +- packages/nocodb/src/schema/swagger.json | 10 ++ 7 files changed, 121 insertions(+), 105 deletions(-) create mode 100644 packages/nocodb/src/lib/meta/api/tableApis.ts create mode 100644 packages/nocodb/src/lib/services/modelVisibilityService.ts diff --git a/packages/nocodb-sdk/src/lib/Api.ts b/packages/nocodb-sdk/src/lib/Api.ts index 13b1fccb47..c2197a21be 100644 --- a/packages/nocodb-sdk/src/lib/Api.ts +++ b/packages/nocodb-sdk/src/lib/Api.ts @@ -871,6 +871,7 @@ export interface UserInfoType { } export type VisibilityRuleReqType = { + id?: string | null; disabled?: { commenter?: BoolType; creator?: BoolType; diff --git a/packages/nocodb/src/lib/controllers/modelVisibilityController.ts b/packages/nocodb/src/lib/controllers/modelVisibilityController.ts index 0c32f3090b..3f298c0d65 100644 --- a/packages/nocodb/src/lib/controllers/modelVisibilityController.ts +++ b/packages/nocodb/src/lib/controllers/modelVisibilityController.ts @@ -1,122 +1,29 @@ -import Model from '../models/Model'; -import ModelRoleVisibility from '../models/ModelRoleVisibility'; import { Router } from 'express'; -import { Tele } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import { getAjvValidatorMw } from '../meta/api/helpers'; +import { modelVisibilityService } from '../services'; + async function xcVisibilityMetaSetAll(req, res) { - Tele.emit('evt', { evt_type: 'uiAcl:updated' }); - for (const d of req.body) { - for (const role of Object.keys(d.disabled)) { - const dataInDb = await ModelRoleVisibility.get({ - role, - // fk_model_id: d.fk_model_id, - fk_view_id: d.id, - }); - if (dataInDb) { - if (d.disabled[role]) { - if (!dataInDb.disabled) { - await ModelRoleVisibility.update(d.id, role, { - disabled: d.disabled[role], - }); - } - } else { - await dataInDb.delete(); - } - } else if (d.disabled[role]) { - await ModelRoleVisibility.insert({ - fk_view_id: d.id, - disabled: d.disabled[role], - role, - }); - } - } - } - Tele.emit('evt', { evt_type: 'uiAcl:updated' }); + await modelVisibilityService.xcVisibilityMetaSetAll({ + visibilityRule: req.body, + projectId: req.params.projectId, + }); res.json({ msg: 'success' }); } -// @ts-ignore -export async function xcVisibilityMetaGet( - projectId, - _models: Model[] = null, - includeM2M = true - // type: 'table' | 'tableAndViews' | 'views' = 'table' -) { - // todo: move to - const roles = ['owner', 'creator', 'viewer', 'editor', 'commenter', 'guest']; - - const defaultDisabled = roles.reduce((o, r) => ({ ...o, [r]: false }), {}); - - let models = - _models || - (await Model.list({ - project_id: projectId, - base_id: undefined, - })); - - models = includeM2M ? models : (models.filter((t) => !t.mm) as Model[]); - - const result = await models.reduce(async (_obj, model) => { - const obj = await _obj; - // obj[model.id] = { - // tn: model.tn, - // _tn: model._tn, - // order: model.order, - // fk_model_id: model.id, - // id: model.id, - // type: model.type, - // disabled: { ...defaultDisabled } - // }; - // if (type === 'tableAndViews') { - const views = await model.getViews(); - for (const view of views) { - obj[view.id] = { - ptn: model.table_name, - _ptn: model.title, - ptype: model.type, - tn: view.title, - _tn: view.title, - table_meta: model.meta, - ...view, - disabled: { ...defaultDisabled }, - }; - // } - } - - return obj; - }, Promise.resolve({})); - - const disabledList = await ModelRoleVisibility.list(projectId); - - for (const d of disabledList) { - // if (d.fk_model_id) result[d.fk_model_id].disabled[d.role] = !!d.disabled; - // else if (type === 'tableAndViews' && d.fk_view_id) - if (result[d.fk_view_id]) - result[d.fk_view_id].disabled[d.role] = !!d.disabled; - } - - return Object.values(result); - // ?.sort( - // (a: any, b: any) => - // (a.order || 0) - (b.order || 0) || - // (a?._tn || a?.tn)?.localeCompare(b?._tn || b?.tn) - // ); -} - const router = Router({ mergeParams: true }); router.get( '/api/v1/db/meta/projects/:projectId/visibility-rules', metaApiMetrics, ncMetaAclMw(async (req, res) => { res.json( - await xcVisibilityMetaGet( - req.params.projectId, - null, - req.query.includeM2M === true || req.query.includeM2M === 'true' - ) + await modelVisibilityService.xcVisibilityMetaGet({ + projectId: req.params.projectId, + includeM2M: + req.query.includeM2M === true || req.query.includeM2M === 'true', + }) ); }, 'modelVisibilityList') ); diff --git a/packages/nocodb/src/lib/meta/api/tableApis.ts b/packages/nocodb/src/lib/meta/api/tableApis.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 614dfd5ee2..13696db0ce 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -18,3 +18,4 @@ export * as gridViewColumnService from './gridViewColumnService'; export * as viewColumnService from './viewColumnService'; export * as metaDiffService from './metaDiffService'; export * as mapViewService from './mapViewService'; +export * as modelVisibilityService from './modelVisibilityService'; diff --git a/packages/nocodb/src/lib/services/modelVisibilityService.ts b/packages/nocodb/src/lib/services/modelVisibilityService.ts new file mode 100644 index 0000000000..ad71f02871 --- /dev/null +++ b/packages/nocodb/src/lib/services/modelVisibilityService.ts @@ -0,0 +1,97 @@ +import { VisibilityRuleReqType } from 'nocodb-sdk'; +import { NcError } from '../meta/helpers/catchError'; +import Model from '../models/Model'; +import ModelRoleVisibility from '../models/ModelRoleVisibility'; +import { Tele } from 'nc-help'; + +export async function xcVisibilityMetaSetAll(param: { + visibilityRule: VisibilityRuleReqType; + projectId: string; +}) { + Tele.emit('evt', { evt_type: 'uiAcl:updated' }); + for (const d of param.visibilityRule) { + for (const role of Object.keys(d.disabled)) { + const view = await Model.get(d.id); + + if (view.project_id !== param.projectId) { + NcError.badRequest('View does not belong to the project'); + } + + const dataInDb = await ModelRoleVisibility.get({ + role, + fk_view_id: d.id, + }); + if (dataInDb) { + if (d.disabled[role]) { + if (!dataInDb.disabled) { + await ModelRoleVisibility.update(d.id, role, { + disabled: d.disabled[role], + }); + } + } else { + await dataInDb.delete(); + } + } else if (d.disabled[role]) { + await ModelRoleVisibility.insert({ + fk_view_id: d.id, + disabled: d.disabled[role], + role, + }); + } + } + } + Tele.emit('evt', { evt_type: 'uiAcl:updated' }); + + return true; +} + +export async function xcVisibilityMetaGet(param: { + projectId: string; + includeM2M?: boolean; + models?: Model[]; +}) { + const { includeM2M = true, projectId, models: _models } = param ?? {}; + + // todo: move to + const roles = ['owner', 'creator', 'viewer', 'editor', 'commenter', 'guest']; + + const defaultDisabled = roles.reduce((o, r) => ({ ...o, [r]: false }), {}); + + let models = + _models || + (await Model.list({ + project_id: projectId, + base_id: undefined, + })); + + models = includeM2M ? models : (models.filter((t) => !t.mm) as Model[]); + + const result = await models.reduce(async (_obj, model) => { + const obj = await _obj; + + const views = await model.getViews(); + for (const view of views) { + obj[view.id] = { + ptn: model.table_name, + _ptn: model.title, + ptype: model.type, + tn: view.title, + _tn: view.title, + table_meta: model.meta, + ...view, + disabled: { ...defaultDisabled }, + }; + } + + return obj; + }, Promise.resolve({})); + + const disabledList = await ModelRoleVisibility.list(projectId); + + for (const d of disabledList) { + if (result[d.fk_view_id]) + result[d.fk_view_id].disabled[d.role] = !!d.disabled; + } + + return Object.values(result); +} diff --git a/packages/nocodb/src/lib/services/viewService.ts b/packages/nocodb/src/lib/services/viewService.ts index d89d2b91ee..66b3aba3c4 100644 --- a/packages/nocodb/src/lib/services/viewService.ts +++ b/packages/nocodb/src/lib/services/viewService.ts @@ -1,7 +1,7 @@ import { Model, View } from '../models'; import { Tele } from 'nc-help'; import { SharedViewReqType, ViewReqType } from 'nocodb-sdk'; -import { xcVisibilityMetaGet } from '../meta/api/modelVisibilityApis'; +import { xcVisibilityMetaGet } from './modelVisibilityService'; export async function viewList(param: { tableId: string; diff --git a/packages/nocodb/src/schema/swagger.json b/packages/nocodb/src/schema/swagger.json index 50fda41697..59a4d17d6b 100644 --- a/packages/nocodb/src/schema/swagger.json +++ b/packages/nocodb/src/schema/swagger.json @@ -10328,6 +10328,16 @@ "items": { "type": "object", "properties": { + "id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, "disabled": { "type": "object", "properties": { From afd73c02a017510709f1e40822b0d56088e77e83 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Mar 2023 11:52:19 +0530 Subject: [PATCH 10/64] refactor: sharedBaseApi hander Signed-off-by: Pranav C --- packages/nocodb/src/lib/services/index.ts | 1 + .../src/lib/services/sharedBaseService.ts | 108 ++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 packages/nocodb/src/lib/services/sharedBaseService.ts diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 13696db0ce..7efb0a2e74 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -19,3 +19,4 @@ export * as viewColumnService from './viewColumnService'; export * as metaDiffService from './metaDiffService'; export * as mapViewService from './mapViewService'; export * as modelVisibilityService from './modelVisibilityService'; +export * as sharedBaseService from './sharedBaseService'; diff --git a/packages/nocodb/src/lib/services/sharedBaseService.ts b/packages/nocodb/src/lib/services/sharedBaseService.ts new file mode 100644 index 0000000000..637a9e322c --- /dev/null +++ b/packages/nocodb/src/lib/services/sharedBaseService.ts @@ -0,0 +1,108 @@ +import { Tele } from 'nc-help'; +import { v4 as uuidv4 } from 'uuid'; +import Project from '../models/Project'; +import { NcError } from '../meta/helpers/catchError'; +// todo: load from config +const config = { + dashboardPath: '/nc', +}; + +export async function createSharedBaseLink(param:{ + projectId: string; + roles: string; + password: string; + siteUrl: string; +}): Promise { + const project = await Project.get(param.projectId); + + let roles = param?.roles; + if (!roles || (roles !== 'editor' && roles !== 'viewer')) { + roles = 'viewer'; + } + + if (!project) { + NcError.badRequest('Invalid project id'); + } + + const data: any = { + uuid: uuidv4(), + password: param?.password, + roles, + }; + + await Project.update(project.id, data); + + data.url = `${param.siteUrl}${config.dashboardPath}#/nc/base/${data.uuid}`; + delete data.password; + Tele.emit('evt', { evt_type: 'sharedBase:generated-link' }); + return data; +} + +export async function updateSharedBaseLink(param: { + projectId: string; + roles: string; + password: string; + siteUrl: string; +}): Promise { + const project = await Project.get(param.projectId); + + let roles = param.roles; + if (!roles || (roles !== 'editor' && roles !== 'viewer')) { + roles = 'viewer'; + } + + if (!project) { + NcError.badRequest('Invalid project id'); + } + const data: any = { + uuid: project.uuid || uuidv4(), + password: param.password, + roles, + }; + + await Project.update(project.id, data); + + data.url = `${param.siteUrl}${config.dashboardPath}#/nc/base/${data.uuid}`; + delete data.password; + Tele.emit('evt', { evt_type: 'sharedBase:generated-link' }); + return data; +} + +export async function disableSharedBaseLink(param:{ + projectId: string; +}): Promise { + const project = await Project.get(param.projectId); + + if (!project) { + NcError.badRequest('Invalid project id'); + } + const data: any = { + uuid: null, + }; + + await Project.update(project.id, data); + + Tele.emit('evt', { evt_type: 'sharedBase:disable-link' }); + + return { uuid: null } +} + +export async function getSharedBaseLink(param:{ + projectId: string; + siteUrl: string; +}): Promise { + const project = await Project.get(param.projectId); + + if (!project) { + NcError.badRequest('Invalid project id'); + } + const data: any = { + uuid: project.uuid, + roles: project.roles, + }; + if (data.uuid) + data.url = `${param.siteUrl}${config.dashboardPath}#/nc/base/${data.shared_base_id}`; + + return data; +} + From ec0c21b7e7e3d137d8bb26f7d793dd599a69c625 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Mar 2023 12:20:03 +0530 Subject: [PATCH 11/64] refactor: orgUserApi handler Signed-off-by: Pranav C --- packages/nocodb/src/lib/services/index.ts | 1 + .../nocodb/src/lib/services/orgUserService.ts | 266 ++++++++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 packages/nocodb/src/lib/services/orgUserService.ts diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 7efb0a2e74..f487e405ed 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -20,3 +20,4 @@ export * as metaDiffService from './metaDiffService'; export * as mapViewService from './mapViewService'; export * as modelVisibilityService from './modelVisibilityService'; export * as sharedBaseService from './sharedBaseService'; +export * as orgUserService from './orgUserService'; diff --git a/packages/nocodb/src/lib/services/orgUserService.ts b/packages/nocodb/src/lib/services/orgUserService.ts new file mode 100644 index 0000000000..e1986a3be6 --- /dev/null +++ b/packages/nocodb/src/lib/services/orgUserService.ts @@ -0,0 +1,266 @@ +import { + AuditOperationSubTypes, + AuditOperationTypes, + PluginCategory, + UserType, +} from 'nocodb-sdk'; +import { v4 as uuidv4 } from 'uuid'; +import validator from 'validator'; +import { OrgUserRoles } from 'nocodb-sdk'; +import { NC_APP_SETTINGS } from '../constants'; +import { Audit, ProjectUser, Store, SyncSource, User } from '../models'; +import Noco from '../Noco'; +import { MetaTable } from '../utils/globals'; +import { Tele } from 'nc-help'; +import { NcError } from '../meta/helpers/catchError'; +import { extractProps } from '../meta/helpers/extractProps'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import { randomTokenString } from '../meta/helpers/stringHelpers'; +import { sendInviteEmail } from '../meta/api/projectUserApis'; + +export async function userList(param: { + // todo: add better typing + query: Record; +}) { + const { query = {} } = param; + + return new PagedResponseImpl(await User.list(query), { + ...query, + count: await User.count(query), + }); +} + +export async function userUpdate(param: { + // todo: better typing + user: Partial; + userId: string; +}) { + const updateBody = extractProps(param.user, ['roles']); + + const user = await User.get(param.userId); + + if (user.roles.includes(OrgUserRoles.SUPER_ADMIN)) { + NcError.badRequest('Cannot update super admin roles'); + } + + return await User.update(param.userId, { + ...updateBody, + token_version: null, + }); +} + +export async function userDelete(param: { userId: string }) { + const ncMeta = await Noco.ncMeta.startTransaction(); + try { + const user = await User.get(param.userId, ncMeta); + + if (user.roles.includes(OrgUserRoles.SUPER_ADMIN)) { + NcError.badRequest('Cannot delete super admin'); + } + + // delete project user entry and assign to super admin + const projectUsers = await ProjectUser.getProjectsIdList( + param.userId, + ncMeta + ); + + // todo: clear cache + + // TODO: assign super admin as project owner + for (const projectUser of projectUsers) { + await ProjectUser.delete( + projectUser.project_id, + projectUser.fk_user_id, + ncMeta + ); + } + + // delete sync source entry + await SyncSource.deleteByUserId(param.userId, ncMeta); + + // delete user + await User.delete(param.userId, ncMeta); + await ncMeta.commit(); + } catch (e) { + await ncMeta.rollback(e); + throw e; + } + + return true; +} + +export async function userAdd(param: { + user: UserType; + projectId: string; + // todo: refactor + req: any; +}) { + // allow only viewer or creator role + if ( + param.user.roles && + ![OrgUserRoles.VIEWER, OrgUserRoles.CREATOR].includes( + param.user.roles as OrgUserRoles + ) + ) { + NcError.badRequest('Invalid role'); + } + + // extract emails from request body + const emails = (param.user.email || '') + .toLowerCase() + .split(/\s*,\s*/) + .map((v) => v.trim()); + + // check for invalid emails + const invalidEmails = emails.filter((v) => !validator.isEmail(v)); + + if (!emails.length) { + return NcError.badRequest('Invalid email address'); + } + if (invalidEmails.length) { + NcError.badRequest('Invalid email address : ' + invalidEmails.join(', ')); + } + + const invite_token = uuidv4(); + const error = []; + + for (const email of emails) { + // add user to project if user already exist + const user = await User.getByEmail(email); + + if (user) { + NcError.badRequest('User already exist'); + } else { + try { + // create new user with invite token + await User.insert({ + invite_token, + invite_token_expires: new Date(Date.now() + 24 * 60 * 60 * 1000), + email, + roles: param.user.roles || OrgUserRoles.VIEWER, + token_version: randomTokenString(), + }); + + const count = await User.count(); + Tele.emit('evt', { evt_type: 'org:user:invite', count }); + + await Audit.insert({ + op_type: AuditOperationTypes.ORG_USER, + op_sub_type: AuditOperationSubTypes.INVITE, + user: param.req.user.email, + description: `invited ${email} to ${param.projectId} project `, + ip: param.req.clientIp, + }); + // in case of single user check for smtp failure + // and send back token if failed + if ( + emails.length === 1 && + !(await sendInviteEmail(email, invite_token, param.req)) + ) { + return { invite_token, email }; + } else { + sendInviteEmail(email, invite_token, param.req); + } + } catch (e) { + console.log(e); + if (emails.length === 1) { + throw e; + } else { + error.push({ email, error: e.message }); + } + } + } + } + + if (emails.length === 1) { + return { + msg: 'success', + }; + } else { + return { invite_token, emails, error }; + } +} + +export async function userSettings(_param): Promise { + NcError.notImplemented(); +} + +export async function userInviteResend(param: { + userId: string; + req: any; +}): Promise { + const user = await User.get(param.userId); + + if (!user) { + NcError.badRequest(`User with id '${param.userId}' not found`); + } + + const invite_token = uuidv4(); + + await User.update(user.id, { + invite_token, + invite_token_expires: new Date(Date.now() + 24 * 60 * 60 * 1000), + }); + + const pluginData = await Noco.ncMeta.metaGet2(null, null, MetaTable.PLUGIN, { + category: PluginCategory.EMAIL, + active: true, + }); + + if (!pluginData) { + NcError.badRequest( + `No Email Plugin is found. Please go to App Store to configure first or copy the invitation URL to users instead.` + ); + } + + await sendInviteEmail(user.email, invite_token, param.req); + + await Audit.insert({ + op_type: AuditOperationTypes.ORG_USER, + op_sub_type: AuditOperationSubTypes.RESEND_INVITE, + user: user.email, + description: `resent a invite to ${user.email} `, + ip: param.req.clientIp, + }); + + return true; +} + +export async function generateResetUrl(param: { + userId: string; + siteUrl: string; +}) { + const user = await User.get(param.userId); + + if (!user) { + NcError.badRequest(`User with id '${param.userId}' not found`); + } + const token = uuidv4(); + await User.update(user.id, { + email: user.email, + reset_password_token: token, + reset_password_expires: new Date(Date.now() + 60 * 60 * 1000), + token_version: null, + }); + + return { + reset_password_token: token, + reset_password_url: param.siteUrl + `/auth/password/reset/${token}`, + }; +} + +export async function appSettingsGet() { + let settings = {}; + try { + settings = JSON.parse((await Store.get(NC_APP_SETTINGS))?.value); + } catch {} + return settings; +} + +export async function appSettingsSet(param: { settings: any }) { + await Store.saveOrUpdate({ + value: JSON.stringify(param.settings), + key: NC_APP_SETTINGS, + }); + return true; +} From 88365ec68373b8353962fd8d1e54c087e0e6ad2d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Mar 2023 13:23:48 +0530 Subject: [PATCH 12/64] refactor: attachment, orguser, orgtoken, appsettings and projectuser Signed-off-by: Pranav C --- .../lib/controllers/orgLicenseController.ts | 11 +- .../src/lib/controllers/orgUserController.ts | 238 ++----------- .../src/lib/services/attachmentService.ts | 118 +++++++ packages/nocodb/src/lib/services/index.ts | 4 + .../src/lib/services/orgLicenseService.ts | 17 + .../src/lib/services/orgTokenService.ts | 53 +++ .../nocodb/src/lib/services/orgUserService.ts | 2 +- .../src/lib/services/projectUserService.ts | 315 ++++++++++++++++++ 8 files changed, 540 insertions(+), 218 deletions(-) create mode 100644 packages/nocodb/src/lib/services/attachmentService.ts create mode 100644 packages/nocodb/src/lib/services/orgLicenseService.ts create mode 100644 packages/nocodb/src/lib/services/orgTokenService.ts create mode 100644 packages/nocodb/src/lib/services/projectUserService.ts diff --git a/packages/nocodb/src/lib/controllers/orgLicenseController.ts b/packages/nocodb/src/lib/controllers/orgLicenseController.ts index 0281145b7a..02b167c3df 100644 --- a/packages/nocodb/src/lib/controllers/orgLicenseController.ts +++ b/packages/nocodb/src/lib/controllers/orgLicenseController.ts @@ -1,21 +1,16 @@ import { Router } from 'express'; import { OrgUserRoles } from 'nocodb-sdk'; -import { NC_LICENSE_KEY } from '../constants'; -import Store from '../models/Store'; -import Noco from '../Noco'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { getAjvValidatorMw } from '../meta/api/helpers'; +import { orgLicenseService } from '../services'; async function licenseGet(_req, res) { - const license = await Store.get(NC_LICENSE_KEY); - - res.json({ key: license?.value }); + res.json(await orgLicenseService.licenseGet()); } async function licenseSet(req, res) { - await Store.saveOrUpdate({ value: req.body.key, key: NC_LICENSE_KEY }); - await Noco.loadEEState(); + await orgLicenseService.licenseSet({ key: req.body.key }) res.json({ msg: 'License key saved' }); } diff --git a/packages/nocodb/src/lib/controllers/orgUserController.ts b/packages/nocodb/src/lib/controllers/orgUserController.ts index 5da0bccee9..c89e031124 100644 --- a/packages/nocodb/src/lib/controllers/orgUserController.ts +++ b/packages/nocodb/src/lib/controllers/orgUserController.ts @@ -1,255 +1,75 @@ import { Router } from 'express'; -import { - AuditOperationSubTypes, - AuditOperationTypes, - PluginCategory, -} from 'nocodb-sdk'; -import { v4 as uuidv4 } from 'uuid'; -import validator from 'validator'; import { OrgUserRoles } from 'nocodb-sdk'; -import { NC_APP_SETTINGS } from '../constants'; -import Audit from '../models/Audit'; -import ProjectUser from '../models/ProjectUser'; -import Store from '../models/Store'; -import SyncSource from '../models/SyncSource'; -import User from '../models/User'; -import Noco from '../Noco'; -import { MetaTable } from '../utils/globals'; -import { Tele } from 'nc-help'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { NcError } from '../meta/helpers/catchError'; -import { extractProps } from '../meta/helpers/extractProps'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import { randomTokenString } from '../meta/helpers/stringHelpers'; import { getAjvValidatorMw } from '../meta/api/helpers'; -import { sendInviteEmail } from './projectUserController'; +import { orgUserService } from '../services'; async function userList(req, res) { res.json( - new PagedResponseImpl(await User.list(req.query), { - ...req.query, - count: await User.count(req.query), + await orgUserService.userList({ + query: req.query, }) ); } async function userUpdate(req, res) { - const updateBody = extractProps(req.body, ['roles']); - - const user = await User.get(req.params.userId); - - if (user.roles.includes(OrgUserRoles.SUPER_ADMIN)) { - NcError.badRequest('Cannot update super admin roles'); - } - res.json( - await User.update(req.params.userId, { - ...updateBody, - token_version: null, + await orgUserService.userUpdate({ + user: req.body, + userId: req.params.userId, }) ); } async function userDelete(req, res) { - const ncMeta = await Noco.ncMeta.startTransaction(); - try { - const user = await User.get(req.params.userId, ncMeta); - - if (user.roles.includes(OrgUserRoles.SUPER_ADMIN)) { - NcError.badRequest('Cannot delete super admin'); - } - - // delete project user entry and assign to super admin - const projectUsers = await ProjectUser.getProjectsIdList( - req.params.userId, - ncMeta - ); - - // todo: clear cache - - // TODO: assign super admin as project owner - for (const projectUser of projectUsers) { - await ProjectUser.delete( - projectUser.project_id, - projectUser.fk_user_id, - ncMeta - ); - } - - // delete sync source entry - await SyncSource.deleteByUserId(req.params.userId, ncMeta); - - // delete user - await User.delete(req.params.userId, ncMeta); - await ncMeta.commit(); - } catch (e) { - await ncMeta.rollback(e); - throw e; - } - + await orgUserService.userDelete({ + userId: req.params.userId, + }); res.json({ msg: 'success' }); } -async function userAdd(req, res, next) { - // allow only viewer or creator role - if ( - req.body.roles && - ![OrgUserRoles.VIEWER, OrgUserRoles.CREATOR].includes(req.body.roles) - ) { - NcError.badRequest('Invalid role'); - } - - // extract emails from request body - const emails = (req.body.email || '') - .toLowerCase() - .split(/\s*,\s*/) - .map((v) => v.trim()); - - // check for invalid emails - const invalidEmails = emails.filter((v) => !validator.isEmail(v)); - - if (!emails.length) { - return NcError.badRequest('Invalid email address'); - } - if (invalidEmails.length) { - NcError.badRequest('Invalid email address : ' + invalidEmails.join(', ')); - } - - const invite_token = uuidv4(); - const error = []; - - for (const email of emails) { - // add user to project if user already exist - const user = await User.getByEmail(email); - - if (user) { - NcError.badRequest('User already exist'); - } else { - try { - // create new user with invite token - await User.insert({ - invite_token, - invite_token_expires: new Date(Date.now() + 24 * 60 * 60 * 1000), - email, - roles: req.body.roles || OrgUserRoles.VIEWER, - token_version: randomTokenString(), - }); - - const count = await User.count(); - Tele.emit('evt', { evt_type: 'org:user:invite', count }); - - await Audit.insert({ - op_type: AuditOperationTypes.ORG_USER, - op_sub_type: AuditOperationSubTypes.INVITE, - user: req.user.email, - description: `invited ${email} to ${req.params.projectId} project `, - ip: req.clientIp, - }); - // in case of single user check for smtp failure - // and send back token if failed - if ( - emails.length === 1 && - !(await sendInviteEmail(email, invite_token, req)) - ) { - return res.json({ invite_token, email }); - } else { - sendInviteEmail(email, invite_token, req); - } - } catch (e) { - console.log(e); - if (emails.length === 1) { - return next(e); - } else { - error.push({ email, error: e.message }); - } - } - } - } +async function userAdd(req, res) { + const result = await orgUserService.userAdd({ + user: req.body, + req, + projectId: req.params.projectId, + }); - if (emails.length === 1) { - res.json({ - msg: 'success', - }); - } else { - return res.json({ invite_token, emails, error }); - } + res.json(result); } -async function userSettings(_req, _res): Promise { - NcError.notImplemented(); +async function userSettings(_req, res): Promise { + await orgUserService.userSettings({}); + res.json({}); } async function userInviteResend(req, res): Promise { - const user = await User.get(req.params.userId); - - if (!user) { - NcError.badRequest(`User with id '${req.params.userId}' not found`); - } - - const invite_token = uuidv4(); - - await User.update(user.id, { - invite_token, - invite_token_expires: new Date(Date.now() + 24 * 60 * 60 * 1000), - }); - - const pluginData = await Noco.ncMeta.metaGet2(null, null, MetaTable.PLUGIN, { - category: PluginCategory.EMAIL, - active: true, - }); - - if (!pluginData) { - NcError.badRequest( - `No Email Plugin is found. Please go to App Store to configure first or copy the invitation URL to users instead.` - ); - } - - await sendInviteEmail(user.email, invite_token, req); - - await Audit.insert({ - op_type: AuditOperationTypes.ORG_USER, - op_sub_type: AuditOperationSubTypes.RESEND_INVITE, - user: user.email, - description: `resent a invite to ${user.email} `, - ip: req.clientIp, + await orgUserService.userInviteResend({ + userId: req.params.userId, + req, }); res.json({ msg: 'success' }); } async function generateResetUrl(req, res) { - const user = await User.get(req.params.userId); - - if (!user) { - NcError.badRequest(`User with id '${req.params.userId}' not found`); - } - const token = uuidv4(); - await User.update(user.id, { - email: user.email, - reset_password_token: token, - reset_password_expires: new Date(Date.now() + 60 * 60 * 1000), - token_version: null, + const result = await orgUserService.generateResetUrl({ + siteUrl: req.ncSiteUrl, + userId: req.params.userId, }); - res.json({ - reset_password_token: token, - reset_password_url: req.ncSiteUrl + `/auth/password/reset/${token}`, - }); + res.json(result); } async function appSettingsGet(_req, res) { - let settings = {}; - try { - settings = JSON.parse((await Store.get(NC_APP_SETTINGS))?.value); - } catch {} + const settings = await orgUserService.appSettingsGet(); res.json(settings); } async function appSettingsSet(req, res) { - await Store.saveOrUpdate({ - value: JSON.stringify(req.body), - key: NC_APP_SETTINGS, + await orgUserService.appSettingsSet({ + settings: req.body, }); res.json({ msg: 'Settings saved' }); diff --git a/packages/nocodb/src/lib/services/attachmentService.ts b/packages/nocodb/src/lib/services/attachmentService.ts new file mode 100644 index 0000000000..7b2b6d3169 --- /dev/null +++ b/packages/nocodb/src/lib/services/attachmentService.ts @@ -0,0 +1,118 @@ +// @ts-ignore +import { Request, Response, Router } from 'express'; +import { nanoid } from 'nanoid'; +import path from 'path'; +import slash from 'slash'; +import mimetypes, { mimeIcons } from '../utils/mimeTypes'; +import { Tele } from 'nc-help'; +import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; +import Local from '../v1-legacy/plugins/adapters/storage/Local'; + +export async function upload(param: { + path?: string; + // todo: proper type + files: unknown[]; +}) { + const filePath = sanitizeUrlPath(param.path?.toString()?.split('/') || ['']); + const destPath = path.join('nc', 'uploads', ...filePath); + + const storageAdapter = await NcPluginMgrv2.storageAdapter(); + + const attachments = await Promise.all( + param.files?.map(async (file: any) => { + const fileName = `${nanoid(18)}${path.extname(file.originalname)}`; + + const url = await storageAdapter.fileCreate( + slash(path.join(destPath, fileName)), + file + ); + + let attachmentPath; + + // if `url` is null, then it is local attachment + if (!url) { + // then store the attachement path only + // url will be constructued in `useAttachmentCell` + attachmentPath = `download/${filePath.join('/')}/${fileName}`; + } + + return { + ...(url ? { url } : {}), + ...(attachmentPath ? { path: attachmentPath } : {}), + title: file.originalname, + mimetype: file.mimetype, + size: file.size, + icon: mimeIcons[path.extname(file.originalname).slice(1)] || undefined, + }; + }) + ); + + Tele.emit('evt', { evt_type: 'image:uploaded' }); + + return attachments; +} + +export async function uploadViaURL(param: { + path?: string; + urls: { + url: string; + fileName: string; + mimetype?: string; + size?: string | number; + }[]; +}) { + const filePath = sanitizeUrlPath(param?.path?.toString()?.split('/') || ['']); + const destPath = path.join('nc', 'uploads', ...filePath); + + const storageAdapter = await NcPluginMgrv2.storageAdapter(); + + const attachments = await Promise.all( + param.urls?.map?.(async (urlMeta) => { + const { url, fileName: _fileName } = urlMeta; + + const fileName = `${nanoid(18)}${_fileName || url.split('/').pop()}`; + + const attachmentUrl = await (storageAdapter as any).fileCreateByUrl( + slash(path.join(destPath, fileName)), + url + ); + + let attachmentPath; + + // if `attachmentUrl` is null, then it is local attachment + if (!attachmentUrl) { + // then store the attachement path only + // url will be constructued in `useAttachmentCell` + attachmentPath = `download/${filePath.join('/')}/${fileName}`; + } + + return { + ...(attachmentUrl ? { url: attachmentUrl } : {}), + ...(attachmentPath ? { path: attachmentPath } : {}), + title: fileName, + mimetype: urlMeta.mimetype, + size: urlMeta.size, + icon: mimeIcons[path.extname(fileName).slice(1)] || undefined, + }; + }) + ); + + Tele.emit('evt', { evt_type: 'image:uploaded' }); + + return attachments; +} + +export async function fileRead(param: { path: string }) { + // get the local storage adapter to display local attachments + const storageAdapter = new Local(); + const type = + mimetypes[path.extname(param.path).split('/').pop().slice(1)] || + 'text/plain'; + + const img = await storageAdapter.fileRead(slash(param.path)); + return { img, type }; +} + +export function sanitizeUrlPath(paths) { + return paths.map((url) => url.replace(/[/.?#]+/g, '_')); +} diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index f487e405ed..9873bb2e11 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -21,3 +21,7 @@ export * as mapViewService from './mapViewService'; export * as modelVisibilityService from './modelVisibilityService'; export * as sharedBaseService from './sharedBaseService'; export * as orgUserService from './orgUserService'; +export * as orgTokenService from './orgTokenService'; +export * as orgLicenseService from './orgLicenseService'; +export * as projectUserService from './projectUserService'; +export * as attachmentService from './attachmentService'; diff --git a/packages/nocodb/src/lib/services/orgLicenseService.ts b/packages/nocodb/src/lib/services/orgLicenseService.ts new file mode 100644 index 0000000000..7de1f717bf --- /dev/null +++ b/packages/nocodb/src/lib/services/orgLicenseService.ts @@ -0,0 +1,17 @@ +import { NC_LICENSE_KEY } from '../constants'; +import Store from '../models/Store'; +import Noco from '../Noco'; + +export async function licenseGet() { + const license = await Store.get(NC_LICENSE_KEY); + + return { key: license?.value } +} + +export async function licenseSet(param:{ + key: string +}) { + await Store.saveOrUpdate({ value: param.key, key: NC_LICENSE_KEY }); + await Noco.loadEEState(); + return true +} diff --git a/packages/nocodb/src/lib/services/orgTokenService.ts b/packages/nocodb/src/lib/services/orgTokenService.ts new file mode 100644 index 0000000000..ede91f0bc6 --- /dev/null +++ b/packages/nocodb/src/lib/services/orgTokenService.ts @@ -0,0 +1,53 @@ +import { ApiTokenReqType, OrgUserRoles } from 'nocodb-sdk'; +import { User } from '../models'; +import ApiToken from '../models/ApiToken'; +import { Tele } from 'nc-help'; +import { NcError } from '../meta/helpers/catchError'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; + +export async function apiTokenList(param: { user: User; query: any }) { + const fk_user_id = param.user.id; + let includeUnmappedToken = false; + if (param.user.roles.includes(OrgUserRoles.SUPER_ADMIN)) { + includeUnmappedToken = true; + } + + return new PagedResponseImpl( + await ApiToken.listWithCreatedBy({ + ...param.query, + fk_user_id, + includeUnmappedToken, + }), + { + ...param.query, + count: await ApiToken.count({ + includeUnmappedToken, + fk_user_id, + }), + } + ); +} + +export async function apiTokenCreate(param: { + user: User; + apiToken: ApiTokenReqType; +}) { + Tele.emit('evt', { evt_type: 'org:apiToken:created' }); + return await ApiToken.insert({ + ...param.apiToken, + fk_user_id: param['user'].id, + }); +} + +export async function apiTokenDelete(param: { user: User; token: string }) { + const fk_user_id = param.user.id; + const apiToken = await ApiToken.getByToken(param.token); + if ( + !param.user.roles.includes(OrgUserRoles.SUPER_ADMIN) && + apiToken.fk_user_id !== fk_user_id + ) { + NcError.notFound('Token not found'); + } + Tele.emit('evt', { evt_type: 'org:apiToken:deleted' }); + return await ApiToken.delete(param.token); +} diff --git a/packages/nocodb/src/lib/services/orgUserService.ts b/packages/nocodb/src/lib/services/orgUserService.ts index e1986a3be6..f58e85f890 100644 --- a/packages/nocodb/src/lib/services/orgUserService.ts +++ b/packages/nocodb/src/lib/services/orgUserService.ts @@ -16,7 +16,7 @@ import { NcError } from '../meta/helpers/catchError'; import { extractProps } from '../meta/helpers/extractProps'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import { randomTokenString } from '../meta/helpers/stringHelpers'; -import { sendInviteEmail } from '../meta/api/projectUserApis'; +import { sendInviteEmail } from './projectUserService'; export async function userList(param: { // todo: add better typing diff --git a/packages/nocodb/src/lib/services/projectUserService.ts b/packages/nocodb/src/lib/services/projectUserService.ts new file mode 100644 index 0000000000..1ee50e9410 --- /dev/null +++ b/packages/nocodb/src/lib/services/projectUserService.ts @@ -0,0 +1,315 @@ +import { OrgUserRoles, ProjectUserReqType } from 'nocodb-sdk'; +import { Tele } from 'nc-help'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import ProjectUser from '../models/ProjectUser'; +import validator from 'validator'; +import { NcError } from '../meta/helpers/catchError'; +import { v4 as uuidv4 } from 'uuid'; +import User from '../models/User'; +import Audit from '../models/Audit'; +import NocoCache from '../cache/NocoCache'; +import { CacheGetType, CacheScope, MetaTable } from '../utils/globals'; +import * as ejs from 'ejs'; +import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; +import Noco from '../Noco'; +import { PluginCategory } from 'nocodb-sdk'; +import { randomTokenString } from '../meta/helpers/stringHelpers'; + +export async function userList(param: { projectId: string; query: any }) { + return new PagedResponseImpl( + await ProjectUser.getUsersList({ + ...param.query, + project_id: param.projectId, + }), + { + ...param.query, + count: await ProjectUser.getUsersCount(param.query), + } + ); +} + +export async function userInvite(param: { + projectId: string; + projectUser: ProjectUserReqType; + req: any; +}): Promise { + const emails = (param.projectUser.email || '') + .toLowerCase() + .split(/\s*,\s*/) + .map((v) => v.trim()); + + // check for invalid emails + const invalidEmails = emails.filter((v) => !validator.isEmail(v)); + if (!emails.length) { + return NcError.badRequest('Invalid email address'); + } + if (invalidEmails.length) { + NcError.badRequest('Invalid email address : ' + invalidEmails.join(', ')); + } + + const invite_token = uuidv4(); + const error = []; + + for (const email of emails) { + // add user to project if user already exist + const user = await User.getByEmail(email); + + if (user) { + // check if this user has been added to this project + const projectUser = await ProjectUser.get(param.projectId, user.id); + if (projectUser) { + NcError.badRequest( + `${user.email} with role ${projectUser.roles} already exists in this project` + ); + } + + await ProjectUser.insert({ + project_id: param.projectId, + fk_user_id: user.id, + roles: param.projectUser.roles || 'editor', + }); + + const cachedUser = await NocoCache.get( + `${CacheScope.USER}:${email}___${param.projectId}`, + CacheGetType.TYPE_OBJECT + ); + + if (cachedUser) { + cachedUser.roles = param.projectUser.roles || 'editor'; + await NocoCache.set( + `${CacheScope.USER}:${email}___${param.projectId}`, + cachedUser + ); + } + + await Audit.insert({ + project_id: param.projectId, + op_type: 'AUTHENTICATION', + op_sub_type: 'INVITE', + user: param.req.user.email, + description: `invited ${email} to ${param.projectId} project `, + ip: param.req.clientIp, + }); + } else { + try { + // create new user with invite token + const { id } = await User.insert({ + invite_token, + invite_token_expires: new Date(Date.now() + 24 * 60 * 60 * 1000), + email, + roles: OrgUserRoles.VIEWER, + token_version: randomTokenString(), + }); + + // add user to project + await ProjectUser.insert({ + project_id: param.projectId, + fk_user_id: id, + roles: param.projectUser.roles, + }); + + const count = await User.count(); + Tele.emit('evt', { evt_type: 'project:invite', count }); + + await Audit.insert({ + project_id: param.projectId, + op_type: 'AUTHENTICATION', + op_sub_type: 'INVITE', + user: param.req.user.email, + description: `invited ${email} to ${param.projectId} project `, + ip: param.req.clientIp, + }); + // in case of single user check for smtp failure + // and send back token if failed + if ( + emails.length === 1 && + !(await sendInviteEmail(email, invite_token, param.req)) + ) { + return { invite_token, email }; + } else { + sendInviteEmail(email, invite_token, param.req); + } + } catch (e) { + console.log(e); + if (emails.length === 1) { + throw e; + } else { + error.push({ email, error: e.message }); + } + } + } + } + + if (emails.length === 1) { + return { + msg: 'success', + }; + } else { + return { invite_token, emails, error }; + } +} + +export async function projectUserUpdate(param: { + userId: string; + // todo: update swagger + projectUser: ProjectUserReqType & { project_id: string }; + // todo: refactor + req: any; + projectId: string; +}): Promise { + // todo: use param.projectId + if (!param.projectUser?.project_id) { + NcError.badRequest('Missing project id in request body.'); + } + + if ( + param.req.session?.passport?.user?.roles?.owner && + param.req.session?.passport?.user?.id === param.userId && + param.projectUser.roles.indexOf('owner') === -1 + ) { + NcError.badRequest("Super admin can't remove Super role themselves"); + } + const user = await User.get(param.userId); + + if (!user) { + NcError.badRequest(`User with id '${param.userId}' doesn't exist`); + } + + // todo: handle roles which contains super + if ( + !param.req.session?.passport?.user?.roles?.owner && + param.projectUser.roles.indexOf('owner') > -1 + ) { + NcError.forbidden('Insufficient privilege to add super admin role.'); + } + + await ProjectUser.update( + param.projectId, + param.userId, + param.projectUser.roles + ); + + await Audit.insert({ + op_type: 'AUTHENTICATION', + op_sub_type: 'ROLES_MANAGEMENT', + user: param.req.user.email, + description: `updated roles for ${user.email} with ${param.projectUser.roles} `, + ip: param.req.clientIp, + }); + + return { + msg: 'User details updated successfully', + }; +} + +export async function projectUserDelete(param: { + projectId: string; + userId: string; + // todo: refactor + req: any; +}): Promise { + const project_id = param.projectId; + + if (param.req.session?.passport?.user?.id === param.userId) { + NcError.badRequest("Admin can't delete themselves!"); + } + + if (!param.req.session?.passport?.user?.roles?.owner) { + const user = await User.get(param.userId); + if (user.roles?.split(',').includes('super')) + NcError.forbidden('Insufficient privilege to delete a super admin user.'); + + const projectUser = await ProjectUser.get(project_id, param.userId); + if (projectUser?.roles?.split(',').includes('super')) + NcError.forbidden('Insufficient privilege to delete a owner user.'); + } + + await ProjectUser.delete(project_id, param.userId); + return true; +} + +export async function projectUserInviteResend(param: { + userId: string; + projectUser: ProjectUserReqType; + projectId: string; + // todo: refactor + req: any; +}): Promise { + const user = await User.get(param.userId); + + if (!user) { + NcError.badRequest(`User with id '${param.userId}' not found`); + } + + param.projectUser.roles = user.roles; + const invite_token = uuidv4(); + + await User.update(user.id, { + invite_token, + invite_token_expires: new Date(Date.now() + 24 * 60 * 60 * 1000), + }); + + const pluginData = await Noco.ncMeta.metaGet2(null, null, MetaTable.PLUGIN, { + category: PluginCategory.EMAIL, + active: true, + }); + + if (!pluginData) { + NcError.badRequest( + `No Email Plugin is found. Please go to App Store to configure first or copy the invitation URL to users instead.` + ); + } + + await sendInviteEmail(user.email, invite_token, param.req); + + await Audit.insert({ + op_type: 'AUTHENTICATION', + op_sub_type: 'RESEND_INVITE', + user: user.email, + description: `resent a invite to ${user.email} `, + ip: param.req.clientIp, + project_id: param.projectId, + }); + + return true; +} + +// todo: refactor the whole function +export async function sendInviteEmail( + email: string, + token: string, + req: any +): Promise { + try { + const template = ( + await import('../meta/api/userApi/ui/emailTemplates/invite') + ).default; + + const emailAdapter = await NcPluginMgrv2.emailAdapter(); + + if (emailAdapter) { + await emailAdapter.mailSend({ + to: email, + subject: 'Verify email', + html: ejs.render(template, { + signupLink: `${req.ncSiteUrl}${ + Noco.getConfig()?.dashboardPath + }#/signup/${token}`, + projectName: req.body?.projectName, + roles: (req.body?.roles || '') + .split(',') + .map((r) => r.replace(/^./, (m) => m.toUpperCase())) + .join(', '), + adminEmail: req.session?.passport?.user?.email, + }), + }); + return true; + } + } catch (e) { + console.log( + 'Warning : `mailSend` failed, Please configure emailClient configuration.', + e.message + ); + throw e; + } +} From 8ca0095a302e9036c3c893080d605f894f95c54b Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Mar 2023 13:49:27 +0530 Subject: [PATCH 13/64] refactor: hook filter api Signed-off-by: Pranav C --- .../lib/controllers/attachmentController.ts | 152 +++--------------- .../src/lib/services/hookFilterService.ts | 61 +++++++ packages/nocodb/src/lib/services/index.ts | 1 + 3 files changed, 88 insertions(+), 126 deletions(-) create mode 100644 packages/nocodb/src/lib/services/hookFilterService.ts diff --git a/packages/nocodb/src/lib/controllers/attachmentController.ts b/packages/nocodb/src/lib/controllers/attachmentController.ts index 2255197ff8..8b9bd0eff7 100644 --- a/packages/nocodb/src/lib/controllers/attachmentController.ts +++ b/packages/nocodb/src/lib/controllers/attachmentController.ts @@ -1,22 +1,16 @@ -// @ts-ignore import { Request, Response, Router } from 'express'; import multer from 'multer'; -import { nanoid } from 'nanoid'; import { OrgUserRoles, ProjectRoles } from 'nocodb-sdk'; import path from 'path'; -import slash from 'slash'; import Noco from '../Noco'; import { MetaTable } from '../utils/globals'; -import mimetypes, { mimeIcons } from '../utils/mimeTypes'; -import { Tele } from 'nc-help'; import extractProjectIdAndAuthenticate from '../meta/helpers/extractProjectIdAndAuthenticate'; import catchError, { NcError } from '../meta/helpers/catchError'; -import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; -import Local from '../v1-legacy/plugins/adapters/storage/Local'; import { NC_ATTACHMENT_FIELD_SIZE } from '../constants'; import { getCacheMiddleware } from '../meta/api/helpers'; +import { attachmentService } from '../services'; -const isUploadAllowed = async (req: Request, _res: Response, next: any) => { +const isUploadAllowedMw = async (req: Request, _res: Response, next: any) => { if (!req['user']?.id) { if (!req['user']?.isPublicBase) { NcError.unauthorized('Unauthorized'); @@ -47,111 +41,29 @@ const isUploadAllowed = async (req: Request, _res: Response, next: any) => { }; export async function upload(req: Request, res: Response) { - const filePath = sanitizeUrlPath( - req.query?.path?.toString()?.split('/') || [''] - ); - const destPath = path.join('nc', 'uploads', ...filePath); - - const storageAdapter = await NcPluginMgrv2.storageAdapter(); - - const attachments = await Promise.all( - (req as any).files?.map(async (file) => { - const fileName = `${nanoid(18)}${path.extname(file.originalname)}`; - - const url = await storageAdapter.fileCreate( - slash(path.join(destPath, fileName)), - file - ); - - let attachmentPath; - - // if `url` is null, then it is local attachment - if (!url) { - // then store the attachement path only - // url will be constructued in `useAttachmentCell` - attachmentPath = `download/${filePath.join('/')}/${fileName}`; - } - - return { - ...(url ? { url } : {}), - ...(attachmentPath ? { path: attachmentPath } : {}), - title: file.originalname, - mimetype: file.mimetype, - size: file.size, - icon: mimeIcons[path.extname(file.originalname).slice(1)] || undefined, - }; - }) - ); - - Tele.emit('evt', { evt_type: 'image:uploaded' }); + const attachments = await attachmentService.upload({ + files: (req as any).files, + path: req.query?.path as string, + }); res.json(attachments); } export async function uploadViaURL(req: Request, res: Response) { - const filePath = sanitizeUrlPath( - req.query?.path?.toString()?.split('/') || [''] - ); - const destPath = path.join('nc', 'uploads', ...filePath); - - const storageAdapter = await NcPluginMgrv2.storageAdapter(); - - const attachments = await Promise.all( - req.body?.map?.(async (urlMeta) => { - const { url, fileName: _fileName } = urlMeta; - - const fileName = `${nanoid(18)}${_fileName || url.split('/').pop()}`; - - const attachmentUrl = await (storageAdapter as any).fileCreateByUrl( - slash(path.join(destPath, fileName)), - url - ); - - let attachmentPath; - - // if `attachmentUrl` is null, then it is local attachment - if (!attachmentUrl) { - // then store the attachement path only - // url will be constructued in `useAttachmentCell` - attachmentPath = `download/${filePath.join('/')}/${fileName}`; - } - - return { - ...(attachmentUrl ? { url: attachmentUrl } : {}), - ...(attachmentPath ? { path: attachmentPath } : {}), - title: fileName, - mimetype: urlMeta.mimetype, - size: urlMeta.size, - icon: mimeIcons[path.extname(fileName).slice(1)] || undefined, - }; - }) - ); - - Tele.emit('evt', { evt_type: 'image:uploaded' }); + const attachments = await attachmentService.uploadViaURL({ + urls: req.body, + path: req.query?.path as string, + }); res.json(attachments); } export async function fileRead(req, res) { try { - // get the local storage adapter to display local attachments - const storageAdapter = new Local(); - const type = - mimetypes[path.extname(req.params?.[0]).split('/').pop().slice(1)] || - 'text/plain'; - - const img = await storageAdapter.fileRead( - slash( - path.join( - 'nc', - 'uploads', - req.params?.[0] - ?.split('/') - .filter((p) => p !== '..') - .join('/') - ) - ) - ); + const { img, type } = await attachmentService.fileRead({ + path: path.join('nc', 'uploads', req.params?.[0]), + }); + res.writeHead(200, { 'Content-Type': type }); res.end(img, 'binary'); } catch (e) { @@ -167,24 +79,16 @@ router.get( getCacheMiddleware(), async (req, res) => { try { - // const type = mimetypes[path.extname(req.params.fileName).slice(1)] || 'text/plain'; - const type = - mimetypes[path.extname(req.params[2]).split('/').pop().slice(1)] || - 'text/plain'; - - const storageAdapter = await NcPluginMgrv2.storageAdapter(); - // const img = await this.storageAdapter.fileRead(slash(path.join('nc', req.params.projectId, req.params.dbAlias, 'uploads', req.params.fileName))); - const img = await storageAdapter.fileRead( - slash( - path.join( - 'nc', - req.params[0], - req.params[1], - 'uploads', - ...req.params[2].split('/') - ) - ) - ); + const { img, type } = await attachmentService.fileRead({ + path: path.join( + 'nc', + req.params[0], + req.params[1], + 'uploads', + ...req.params[2].split('/') + ), + }); + res.writeHead(200, { 'Content-Type': type }); res.end(img, 'binary'); } catch (e) { @@ -193,10 +97,6 @@ router.get( } ); -export function sanitizeUrlPath(paths) { - return paths.map((url) => url.replace(/[/.?#]+/g, '_')); -} - router.post( '/api/v1/db/storage/upload', multer({ @@ -207,7 +107,7 @@ router.post( }).any(), [ extractProjectIdAndAuthenticate, - catchError(isUploadAllowed), + catchError(isUploadAllowedMw), catchError(upload), ] ); @@ -217,7 +117,7 @@ router.post( [ extractProjectIdAndAuthenticate, - catchError(isUploadAllowed), + catchError(isUploadAllowedMw), catchError(uploadViaURL), ] ); diff --git a/packages/nocodb/src/lib/services/hookFilterService.ts b/packages/nocodb/src/lib/services/hookFilterService.ts new file mode 100644 index 0000000000..e8e94d7233 --- /dev/null +++ b/packages/nocodb/src/lib/services/hookFilterService.ts @@ -0,0 +1,61 @@ +import { Tele } from 'nc-help'; +import { FilterReqType } from 'nocodb-sdk'; +import Filter from '../models/Filter'; + +export async function filterGet(param: { hookId: string }) { + const filter = await Filter.getFilterObject({ hookId: param.hookId }); + + return filter; +} + +export async function filterList(param: { hookId: string }) { + const filters = await Filter.rootFilterListByHook({ + hookId: param.hookId, + }); + + return filters; +} + +export async function filterChildrenRead(param: { + hookId: string; + filterParentId: string; +}) { + const filter = await Filter.parentFilterListByHook({ + hookId: param.hookId, + parentId: param.filterParentId, + }); + + return filter; +} + +export async function filterCreate(param: { + hookId: string; + filter: FilterReqType; +}) { + const filter = await Filter.insert({ + ...param.filter, + fk_hook_id: param.hookId, + }); + + Tele.emit('evt', { evt_type: 'hookFilter:created' }); + return filter; +} + +export async function filterUpdate(param: { + hookId: string; + filterId: string; + filter: FilterReqType; +}) { + const filter = await Filter.update(param.filterId, { + ...param.filter, + fk_hook_id: param.hookId, + } as Filter); + Tele.emit('evt', { evt_type: 'hookFilter:updated' }); + return filter; +} + +export async function filterDelete(param: { filterId: string }) { + await Filter.delete(param.filterId); + Tele.emit('evt', { evt_type: 'hookFilter:deleted' }); + return true; +} diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 9873bb2e11..12395ce307 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -25,3 +25,4 @@ export * as orgTokenService from './orgTokenService'; export * as orgLicenseService from './orgLicenseService'; export * as projectUserService from './projectUserService'; export * as attachmentService from './attachmentService'; +export * as hookFilterService from './hookFilterService'; From e9da75f91f87bcd2f64e57781d01845c717ad6e9 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Mar 2023 17:53:24 +0530 Subject: [PATCH 14/64] refactor: data alias apis api Signed-off-by: Pranav C --- .../lib/controllers/dataController/index.ts | 258 +++++++++++++ .../src/lib/services/dataService/index.ts | 350 ++++++++++++++++++ packages/nocodb/src/lib/services/index.ts | 1 + 3 files changed, 609 insertions(+) create mode 100644 packages/nocodb/src/lib/controllers/dataController/index.ts create mode 100644 packages/nocodb/src/lib/services/dataService/index.ts diff --git a/packages/nocodb/src/lib/controllers/dataController/index.ts b/packages/nocodb/src/lib/controllers/dataController/index.ts new file mode 100644 index 0000000000..52973d639c --- /dev/null +++ b/packages/nocodb/src/lib/controllers/dataController/index.ts @@ -0,0 +1,258 @@ +import { Request, Response, Router } from 'express'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import apiMetrics from '../../meta/helpers/apiMetrics'; +import { parseHrtimeToSeconds } from '../../meta/api/helpers'; + +import { dataService } from '../../services'; + +// todo: Handle the error case where view doesnt belong to model +async function dataList(req: Request, res: Response) { + const startTime = process.hrtime(); + const responseData = await dataService.dataList({ + query: req.query, + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + }); + const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime)); + res.setHeader('xc-db-response', elapsedSeconds); + res.json(responseData); +} + +async function dataFindOne(req: Request, res: Response) { + res.json( + await dataService.dataFindOne({ + query: req.query, + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + }) + ); +} + +async function dataGroupBy(req: Request, res: Response) { + res.json( + await dataService.dataFindOne({ + query: req.query, + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + }) + ); +} + +async function dataCount(req: Request, res: Response) { + const count = await dataService.dataCount({ + query: req.query, + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + }); + + res.json({ count }); +} + +async function dataInsert(req: Request, res: Response) { + res.json( + await dataService.dataInsert({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + body: req.body, + cookie: req, + }) + ); +} + +async function dataUpdate(req: Request, res: Response) { + res.json( + await dataService.dataUpdate({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + body: req.body, + cookie: req, + rowId: req.params.rowId, + }) + ); +} + +async function dataDelete(req: Request, res: Response) { + res.json( + await dataService.dataDelete({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + cookie: req, + rowId: req.params.rowId, + }) + ); +} + +async function dataRead(req: Request, res: Response) { + res.json( + await dataService.dataRead({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + rowId: req.params.rowId, + query: req.query, + }) +} + +async function dataExist(req: Request, res: Response) { + res.json(await dataService.dataExist({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + rowId: req.params.rowId, + query: req.query, + })); +} + +// todo: Handle the error case where view doesnt belong to model +async function groupedDataList(req: Request, res: Response) { + const startTime = process.hrtime(); + const groupedData = await dataService.groupedDataList({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + query: req.query, + columnId: req.params.columnId + }); + const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime)); + res.setHeader('xc-db-response', elapsedSeconds); + res.json(groupedData); +} + +const router = Router({ mergeParams: true }); + +// table data crud apis +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName', + apiMetrics, + ncMetaAclMw(dataList, 'dataList') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/find-one', + apiMetrics, + ncMetaAclMw(dataFindOne, 'dataFindOne') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/groupby', + apiMetrics, + ncMetaAclMw(dataGroupBy, 'dataGroupBy') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/group/:columnId', + apiMetrics, + ncMetaAclMw(groupedDataList, 'groupedDataList') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/exist', + apiMetrics, + ncMetaAclMw(dataExist, 'dataExist') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/count', + apiMetrics, + ncMetaAclMw(dataCount, 'dataCount') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/count', + apiMetrics, + ncMetaAclMw(dataCount, 'dataCount') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId', + apiMetrics, + ncMetaAclMw(dataRead, 'dataRead') +); + +router.patch( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId', + apiMetrics, + ncMetaAclMw(dataUpdate, 'dataUpdate') +); + +router.delete( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId', + apiMetrics, + ncMetaAclMw(dataDelete, 'dataDelete') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName', + apiMetrics, + ncMetaAclMw(dataList, 'dataList') +); + +// table view data crud apis +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName', + apiMetrics, + ncMetaAclMw(dataList, 'dataList') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/find-one', + apiMetrics, + ncMetaAclMw(dataFindOne, 'dataFindOne') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/groupby', + apiMetrics, + ncMetaAclMw(dataGroupBy, 'dataGroupBy') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/group/:columnId', + apiMetrics, + ncMetaAclMw(groupedDataList, 'groupedDataList') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId/exist', + apiMetrics, + ncMetaAclMw(dataExist, 'dataExist') +); + +router.post( + '/api/v1/db/data/:orgs/:projectName/:tableName', + apiMetrics, + ncMetaAclMw(dataInsert, 'dataInsert') +); + +router.post( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName', + apiMetrics, + ncMetaAclMw(dataInsert, 'dataInsert') +); + +router.patch( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', + apiMetrics, + ncMetaAclMw(dataUpdate, 'dataUpdate') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', + apiMetrics, + ncMetaAclMw(dataRead, 'dataRead') +); + +router.delete( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', + apiMetrics, + ncMetaAclMw(dataDelete, 'dataDelete') +); + +export default router; diff --git a/packages/nocodb/src/lib/services/dataService/index.ts b/packages/nocodb/src/lib/services/dataService/index.ts new file mode 100644 index 0000000000..27575fe156 --- /dev/null +++ b/packages/nocodb/src/lib/services/dataService/index.ts @@ -0,0 +1,350 @@ +import { nocoExecute } from 'nc-help/dist/module/NocoExecute'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import { NcError } from '../../meta/helpers/catchError'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import { Base, Model, View } from '../../models'; +import Project from '../../models/Project'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; + +interface PathParams { + projectName: string; + tableName: string; + viewName: string; +} + +export async function dataList(param: PathParams & { query: any }) { + const { model, view } = await getViewAndModelByAliasOrId(param); + const responseData = await getDataList({ model, view, query: param.query }); + return responseData; +} + +export async function dataFindOne(param: PathParams & { query: any }) { + const { model, view } = await getViewAndModelByAliasOrId(param); + return await getFindOne({ model, view, query: param.query }); +} + +export async function dataGroupBy(param: PathParams & { query: any }) { + const { model, view } = await getViewAndModelByAliasOrId(param); + await getDataGroupBy({ model, view, query: param.query }); +} + +export async function dataCount(param: PathParams & { query: any }) { + const { model, view } = await getViewAndModelByAliasOrId(param); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const countArgs: any = { ...param.query }; + try { + countArgs.filterArr = JSON.parse(countArgs.filterArrJson); + } catch (e) {} + + const count: number = await baseModel.count(countArgs); + + return { count }; +} + +export async function dataInsert( + param: PathParams & { body: unknown; cookie: any } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + await baseModel.insert(param.body, null, param.cookie); +} + +export async function dataUpdate( + param: PathParams & { body: unknown; cookie: any; rowId: string } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + return await baseModel.updateByPk( + param.rowId, + param.body, + null, + param.cookie + ); +} + +export async function dataDelete( + param: PathParams & { rowId: string; cookie: any } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + const base = await Base.get(model.base_id); + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + // todo: Should have error http status code + const message = await baseModel.hasLTARData(param.rowId, model); + if (message.length) { + return { message }; + } + return await baseModel.delByPk(param.rowId, null, param.cookie); +} + +export async function getDataList(param: { + model: Model; + view: View; + query: any; +}) { + const { model, view, query = {} } = param; + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const requestObj = await getAst({ model, query, view }); + + const listArgs: any = { ...query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + + let data = []; + let count = 0; + try { + data = await nocoExecute( + requestObj, + await baseModel.list(listArgs), + {}, + listArgs + ); + count = await baseModel.count(listArgs); + } catch (e) { + console.log(e); + NcError.internalServerError( + 'Internal Server Error, check server log for more details' + ); + } + + return new PagedResponseImpl(data, { + ...query, + count, + }); +} + +export async function getFindOne(param: { + model: Model; + view: View; + query: any; +}) { + const { model, view, query = {} } = param; + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const args: any = { ...query }; + try { + args.filterArr = JSON.parse(args.filterArrJson); + } catch (e) {} + try { + args.sortArr = JSON.parse(args.sortArrJson); + } catch (e) {} + + const data = await baseModel.findOne(args); + return data + ? await nocoExecute( + await getAst({ model, query: args, view }), + data, + {}, + {} + ) + : {}; +} + +export async function getDataGroupBy(param: { + model: Model; + view: View; + query?: any; +}) { + const { model, view, query = {} } = param; + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const listArgs: any = { ...query }; + const data = await baseModel.groupBy({ ...query }); + const count = await baseModel.count(listArgs); + + return new PagedResponseImpl(data, { + ...query, + count, + }); +} + +export async function dataRead( + param: PathParams & { query: any; rowId: string } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const row = await baseModel.readByPk(param.rowId); + + if (!row) { + NcError.notFound(); + } + + return await nocoExecute( + await getAst({ model, query: param.query, view }), + row, + {}, + param.query + ); +} + +export async function dataExist( + param: PathParams & { rowId: string; query: any } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + await baseModel.exist(param.rowId); +} + +// todo: Handle the error case where view doesnt belong to model +export async function groupedDataList( + param: PathParams & { query: any; columnId: string } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + const groupedData = await getGroupedDataList({ + model, + view, + query: param.query, + columnId: param.columnId, + }); + return groupedData; +} + +export async function getGroupedDataList(param: { + model; + view: View; + query: any; + columnId: string; +}) { + const { model, view, query = {} } = param; + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const requestObj = await getAst({ model, query, view }); + + const listArgs: any = { ...query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + try { + listArgs.options = JSON.parse(listArgs.optionsArrJson); + } catch (e) {} + + let data = []; + + const groupedData = await baseModel.groupedList({ + ...listArgs, + groupColumnId: param.columnId, + }); + data = await nocoExecute( + { key: 1, value: requestObj }, + groupedData, + {}, + listArgs + ); + const countArr = await baseModel.groupedListCount({ + ...listArgs, + groupColumnId: param.columnId, + }); + data = data.map((item) => { + // todo: use map to avoid loop + const count = + countArr.find((countItem: any) => countItem.key === item.key)?.count ?? 0; + + item.value = new PagedResponseImpl(item.value, { + ...query, + count: count, + }); + return item; + }); + + return data; +} + +export async function getViewAndModelByAliasOrId(param: { + projectName: string; + tableName: string; + viewName?: string; +}) { + const project = await Project.getWithInfoByTitleOrId(param.projectName); + + const model = await Model.getByAliasOrId({ + project_id: project.id, + aliasOrId: param.tableName, + }); + const view = + param.viewName && + (await View.getByTitleOrId({ + titleOrId: param.viewName, + fk_model_id: model.id, + })); + if (!model) NcError.notFound('Table not found'); + return { model, view }; +} diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 12395ce307..6bcf4667aa 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -26,3 +26,4 @@ export * as orgLicenseService from './orgLicenseService'; export * as projectUserService from './projectUserService'; export * as attachmentService from './attachmentService'; export * as hookFilterService from './hookFilterService'; +export * as dataService from './dataService'; From a49704f373f0099e2b04e82cc97d28c9427233b6 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Mar 2023 18:37:03 +0530 Subject: [PATCH 15/64] refactor: relation data apis and deprecated api Signed-off-by: Pranav C --- .../lib/controllers/dataController/index.ts | 19 +- .../dataController/nestedAndDeprecated.ts | 192 +++++++ .../src/lib/services/dataService/index.ts | 475 +++++++++++++++++- 3 files changed, 677 insertions(+), 9 deletions(-) create mode 100644 packages/nocodb/src/lib/controllers/dataController/nestedAndDeprecated.ts diff --git a/packages/nocodb/src/lib/controllers/dataController/index.ts b/packages/nocodb/src/lib/controllers/dataController/index.ts index 52973d639c..d2c11c8a92 100644 --- a/packages/nocodb/src/lib/controllers/dataController/index.ts +++ b/packages/nocodb/src/lib/controllers/dataController/index.ts @@ -98,16 +98,19 @@ async function dataRead(req: Request, res: Response) { rowId: req.params.rowId, query: req.query, }) + ); } async function dataExist(req: Request, res: Response) { - res.json(await dataService.dataExist({ - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - rowId: req.params.rowId, - query: req.query, - })); + res.json( + await dataService.dataExist({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + rowId: req.params.rowId, + query: req.query, + }) + ); } // todo: Handle the error case where view doesnt belong to model @@ -118,7 +121,7 @@ async function groupedDataList(req: Request, res: Response) { tableName: req.params.tableName, viewName: req.params.viewName, query: req.query, - columnId: req.params.columnId + columnId: req.params.columnId, }); const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime)); res.setHeader('xc-db-response', elapsedSeconds); diff --git a/packages/nocodb/src/lib/controllers/dataController/nestedAndDeprecated.ts b/packages/nocodb/src/lib/controllers/dataController/nestedAndDeprecated.ts new file mode 100644 index 0000000000..6914383321 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/dataController/nestedAndDeprecated.ts @@ -0,0 +1,192 @@ +import { Request, Response, Router } from 'express'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import apiMetrics from '../../meta/helpers/apiMetrics'; +import { dataService } from '../../services'; + +export async function dataList(req: Request, res: Response) { + res.json( + await dataService.dataListByViewId({ + viewId: req.params.viewId, + query: req.query, + }) + ); +} + +export async function mmList(req: Request, res: Response) { + res.json( + await dataService.mmList({ + viewId: req.params.viewId, + colId: req.params.colId, + rowId: req.params.rowId, + query: req.query, + }) + ); +} + +export async function mmExcludedList(req: Request, res: Response) { + res.json( + await dataService.mmExcludedList({ + viewId: req.params.viewId, + colId: req.params.colId, + rowId: req.params.rowId, + query: req.query, + }) + ); +} + +export async function hmExcludedList(req: Request, res: Response) { + res.json( + await dataService.hmExcludedList({ + viewId: req.params.viewId, + colId: req.params.colId, + rowId: req.params.rowId, + query: req.query, + }) + ); +} + +export async function btExcludedList(req: Request, res: Response) { + res.json( + await dataService.btExcludedList({ + viewId: req.params.viewId, + colId: req.params.colId, + rowId: req.params.rowId, + query: req.query, + }) + ); +} + +export async function hmList(req: Request, res: Response) { + res.json( + await dataService.hmList({ + viewId: req.params.viewId, + colId: req.params.colId, + rowId: req.params.rowId, + query: req.query, + }) + ); +} + +async function dataRead(req: Request, res: Response) { + res.json( + await dataService.dataReadByViewId({ + viewId: req.params.viewId, + rowId: req.params.rowId, + query: req.query, + }) + ); +} + +async function dataInsert(req: Request, res: Response) { + res.json( + await dataService.dataInsertByViewId({ + viewId: req.params.viewId, + body: req.body, + cookie: req, + }) + ); +} + +async function dataUpdate(req: Request, res: Response) { + res.json( + await dataService.dataUpdateByViewId({ + viewId: req.params.viewId, + rowId: req.params.rowId, + body: req.body, + cookie: req, + }) + ); +} + +async function dataDelete(req: Request, res: Response) { + res.json( + await dataService.dataDeleteByViewId({ + viewId: req.params.viewId, + rowId: req.params.rowId, + cookie: req, + }) + ); +} + +async function relationDataDelete(req, res) { + await dataService.relationDataDelete({ + viewId: req.params.viewId, + colId: req.params.colId, + childId: req.params.childId, + rowId: req.params.rowId, + cookie: req, + }); + + res.json({ msg: 'success' }); +} + +//@ts-ignore +async function relationDataAdd(req, res) { + await dataService.relationDataAdd({ + viewId: req.params.viewId, + colId: req.params.colId, + childId: req.params.childId, + rowId: req.params.rowId, + cookie: req, + }); + + res.json({ msg: 'success' }); +} + +const router = Router({ mergeParams: true }); + +router.get('/data/:viewId/', apiMetrics, ncMetaAclMw(dataList, 'dataList')); +router.post( + '/data/:viewId/', + apiMetrics, + ncMetaAclMw(dataInsert, 'dataInsert') +); +router.get( + '/data/:viewId/:rowId', + apiMetrics, + ncMetaAclMw(dataRead, 'dataRead') +); +router.patch( + '/data/:viewId/:rowId', + apiMetrics, + ncMetaAclMw(dataUpdate, 'dataUpdate') +); +router.delete( + '/data/:viewId/:rowId', + apiMetrics, + ncMetaAclMw(dataDelete, 'dataDelete') +); + +router.get( + '/data/:viewId/:rowId/mm/:colId', + apiMetrics, + ncMetaAclMw(mmList, 'mmList') +); +router.get( + '/data/:viewId/:rowId/hm/:colId', + apiMetrics, + ncMetaAclMw(hmList, 'hmList') +); + +router.get( + '/data/:viewId/:rowId/mm/:colId/exclude', + ncMetaAclMw(mmExcludedList, 'mmExcludedList') +); +router.get( + '/data/:viewId/:rowId/hm/:colId/exclude', + ncMetaAclMw(hmExcludedList, 'hmExcludedList') +); +router.get( + '/data/:viewId/:rowId/bt/:colId/exclude', + ncMetaAclMw(btExcludedList, 'btExcludedList') +); + +router.post( + '/data/:viewId/:rowId/:relationType/:colId/:childId', + ncMetaAclMw(relationDataAdd, 'relationDataAdd') +); +router.delete( + '/data/:viewId/:rowId/:relationType/:colId/:childId', + ncMetaAclMw(relationDataDelete, 'relationDataDelete') +); +export default router; diff --git a/packages/nocodb/src/lib/services/dataService/index.ts b/packages/nocodb/src/lib/services/dataService/index.ts index 27575fe156..42d7a05556 100644 --- a/packages/nocodb/src/lib/services/dataService/index.ts +++ b/packages/nocodb/src/lib/services/dataService/index.ts @@ -227,7 +227,7 @@ export async function dataRead( const row = await baseModel.readByPk(param.rowId); if (!row) { - NcError.notFound(); + NcError.notFound('Row not found'); } return await nocoExecute( @@ -348,3 +348,476 @@ export async function getViewAndModelByAliasOrId(param: { if (!model) NcError.notFound('Table not found'); return { model, view }; } + +export async function dataListByViewId(param: { viewId: string; query: any }) { + const view = await View.get(param.viewId); + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id || param.viewId, + }); + + if (!model) NcError.notFound('Table not found'); + + return await getDataList({ model, view, query: param.query }); +} + +export async function mmList(param: { + viewId: string; + colId: string; + query: any; + rowId: string; +}) { + const view = await View.get(param.viewId); + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id || param.viewId, + }); + + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const key = `${model.title}List`; + const requestObj: any = { + [key]: 1, + }; + + const data = ( + await nocoExecute( + requestObj, + { + [key]: async (args) => { + return await baseModel.mmList( + { + colId: param.colId, + parentId: param.rowId, + }, + args + ); + }, + }, + {}, + + { nested: { [key]: param.query } } + ) + )?.[key]; + + const count: any = await baseModel.mmListCount({ + colId: param.colId, + parentId: param.rowId, + }); + + return new PagedResponseImpl(data, { + count, + ...param.query, + }); +} + +export async function mmExcludedList(param: { + viewId: string; + colId: string; + query: any; + rowId: string; +}) { + const view = await View.get(param.viewId); + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id || param.viewId, + }); + + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const key = 'List'; + const requestObj: any = { + [key]: 1, + }; + + const data = ( + await nocoExecute( + requestObj, + { + [key]: async (args) => { + return await baseModel.getMmChildrenExcludedList( + { + colId: param.colId, + pid: param.rowId, + }, + args + ); + }, + }, + {}, + + { nested: { [key]: param.query } } + ) + )?.[key]; + + const count = await baseModel.getMmChildrenExcludedListCount( + { + colId: param.colId, + pid: param.rowId, + }, + param.query + ); + + return new PagedResponseImpl(data, { + count, + ...param.query, + }); +} + +export async function hmExcludedList(param: { + viewId: string; + colId: string; + query: any; + rowId: string; +}) { + const view = await View.get(param.viewId); + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id || param.viewId, + }); + + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const key = 'List'; + const requestObj: any = { + [key]: 1, + }; + + const data = ( + await nocoExecute( + requestObj, + { + [key]: async (args) => { + return await baseModel.getHmChildrenExcludedList( + { + colId: param.colId, + pid: param.rowId, + }, + args + ); + }, + }, + {}, + + { nested: { [key]: param.query } } + ) + )?.[key]; + + const count = await baseModel.getHmChildrenExcludedListCount( + { + colId: param.colId, + pid: param.rowId, + }, + param.query + ); + + new PagedResponseImpl(data, { + count, + ...param.query, + }); +} + +export async function btExcludedList(param: { + viewId: string; + colId: string; + query: any; + rowId: string; +}) { + const view = await View.get(param.viewId); + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id || param.viewId, + }); + + if (!model) return NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const key = 'List'; + const requestObj: any = { + [key]: 1, + }; + + const data = ( + await nocoExecute( + requestObj, + { + [key]: async (args) => { + return await baseModel.getBtChildrenExcludedList( + { + colId: param.colId, + cid: param.rowId, + }, + args + ); + }, + }, + {}, + + { nested: { [key]: param.query } } + ) + )?.[key]; + + const count = await baseModel.getBtChildrenExcludedListCount( + { + colId: param.colId, + cid: param.rowId, + }, + param.query + ); + + return new PagedResponseImpl(data, { + count, + ...param.query, + }); +} + +export async function hmList(param: { + viewId: string; + colId: string; + query: any; + rowId: string; +}) { + const view = await View.get(param.viewId); + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id || param.viewId, + }); + + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const key = `${model.title}List`; + const requestObj: any = { + [key]: 1, + }; + + const data = ( + await nocoExecute( + requestObj, + { + [key]: async (args) => { + return await baseModel.hmList( + { + colId: param.colId, + id: param.rowId, + }, + args + ); + }, + }, + {}, + { nested: { [key]: param.query } } + ) + )?.[key]; + + const count = await baseModel.hmListCount({ + colId: param.colId, + id: param.rowId, + }); + + return new PagedResponseImpl(data, { + totalRows: count, + } as any); +} + +export async function dataReadByViewId(param: { + viewId: string; + rowId: string; + query: any; +}) { + try { + const model = await Model.getByIdOrName({ + id: param.viewId, + }); + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + return await nocoExecute( + await getAst({ model, query: param.query }), + await baseModel.readByPk(param.rowId), + {}, + {} + ); + } catch (e) { + console.log(e); + NcError.internalServerError( + 'Internal Server Error, check server log for more details' + ); + } +} + +export async function dataInsertByViewId(param: { + viewId: string; + body: any; + cookie: any; +}) { + const model = await Model.getByIdOrName({ + id: param.viewId, + }); + if (!model) return NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + return await baseModel.insert(param.body, null, param.cookie); +} + +export async function dataUpdateByViewId(param: { + viewId: string; + rowId: string; + body: any; + cookie: any; +}) { + const model = await Model.getByIdOrName({ + id: param.viewId, + }); + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + return await baseModel.updateByPk( + param.rowId, + param.body, + null, + param.cookie + ); +} + +export async function dataDeleteByViewId(param: { + viewId: string; + rowId: string; + cookie: any; +}) { + const model = await Model.getByIdOrName({ + id: param.viewId, + }); + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + return await baseModel.delByPk(param.rowId, null, param.cookie); +} + +export async function relationDataDelete(param: { + viewId: string; + colId: string; + childId: string; + rowId: string; + cookie: any; +}) { + const view = await View.get(param.viewId); + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id || param.viewId, + }); + + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + await baseModel.removeChild({ + colId: param.colId, + childId: param.childId, + rowId: param.rowId, + cookie: param.cookie, + }); + + return true; +} + +export async function relationDataAdd(param: { + viewId: string; + colId: string; + childId: string; + rowId: string; + cookie: any; +}) { + const view = await View.get(param.viewId); + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id || param.viewId, + }); + + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + await baseModel.addChild({ + colId: param.colId, + childId: param.childId, + rowId: param.rowId, + cookie: param.cookie, + }); + + return true; +} From 8c9b85693cab73a4addeaad02059a156deb2b082 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Mar 2023 19:02:06 +0530 Subject: [PATCH 16/64] refactor: bulk data apis Signed-off-by: Pranav C --- .../controllers/dataController/bulkData.ts | 91 ++++++++++++++++ .../src/lib/services/dataService/bulkData.ts | 102 ++++++++++++++++++ .../src/lib/services/dataService/helpers.ts | 31 ++++++ .../src/lib/services/dataService/index.ts | 28 +---- packages/nocodb/src/lib/services/index.ts | 1 + 5 files changed, 226 insertions(+), 27 deletions(-) create mode 100644 packages/nocodb/src/lib/controllers/dataController/bulkData.ts create mode 100644 packages/nocodb/src/lib/services/dataService/bulkData.ts create mode 100644 packages/nocodb/src/lib/services/dataService/helpers.ts diff --git a/packages/nocodb/src/lib/controllers/dataController/bulkData.ts b/packages/nocodb/src/lib/controllers/dataController/bulkData.ts new file mode 100644 index 0000000000..5f169d21e2 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/dataController/bulkData.ts @@ -0,0 +1,91 @@ +import { Request, Response, Router } from 'express'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import apiMetrics from '../../meta/helpers/apiMetrics'; +import { bulkDataService } from '../../services'; + +async function bulkDataInsert(req: Request, res: Response) { + res.json( + await bulkDataService.bulkDataInsert({ + body: req.body, + cookie: req, + projectName: req.params.projectName, + tableName: req.params.tableName, + }) + ); +} + +async function bulkDataUpdate(req: Request, res: Response) { + res.json( + await bulkDataService.bulkDataUpdate({ + body: req.body, + cookie: req, + projectName: req.params.projectName, + tableName: req.params.tableName, + }) + ); +} + +// todo: Integrate with filterArrJson bulkDataUpdateAll +async function bulkDataUpdateAll(req: Request, res: Response) { + res.json( + await bulkDataService.bulkDataUpdateAll({ + body: req.body, + cookie: req, + projectName: req.params.projectName, + tableName: req.params.tableName, + query: req.query, + }) + ); +} + +async function bulkDataDelete(req: Request, res: Response) { + res.json( + await bulkDataService.bulkDataDelete({ + body: req.body, + cookie: req, + projectName: req.params.projectName, + tableName: req.params.tableName, + }) + ); +} + +// todo: Integrate with filterArrJson bulkDataDeleteAll +async function bulkDataDeleteAll(req: Request, res: Response) { + res.json( + await bulkDataService.bulkDataDeleteAll({ + // cookie: req, + projectName: req.params.projectName, + tableName: req.params.tableName, + query: req.query, + }) + ); +} +const router = Router({ mergeParams: true }); + +router.post( + '/api/v1/db/data/bulk/:orgs/:projectName/:tableName', + apiMetrics, + ncMetaAclMw(bulkDataInsert, 'bulkDataInsert') +); +router.patch( + '/api/v1/db/data/bulk/:orgs/:projectName/:tableName', + apiMetrics, + ncMetaAclMw(bulkDataUpdate, 'bulkDataUpdate') +); +router.patch( + '/api/v1/db/data/bulk/:orgs/:projectName/:tableName/all', + apiMetrics, + ncMetaAclMw(bulkDataUpdateAll, 'bulkDataUpdateAll') +); +router.delete( + '/api/v1/db/data/bulk/:orgs/:projectName/:tableName', + apiMetrics, + ncMetaAclMw(bulkDataDelete, 'bulkDataDelete') +); +router.delete( + '/api/v1/db/data/bulk/:orgs/:projectName/:tableName/all', + apiMetrics, + ncMetaAclMw(bulkDataDeleteAll, 'bulkDataDeleteAll') +); + +export default router; diff --git a/packages/nocodb/src/lib/services/dataService/bulkData.ts b/packages/nocodb/src/lib/services/dataService/bulkData.ts new file mode 100644 index 0000000000..2ac6b5a20b --- /dev/null +++ b/packages/nocodb/src/lib/services/dataService/bulkData.ts @@ -0,0 +1,102 @@ +import { BaseModelSqlv2 } from '../../db/sql-data-mapper/lib/sql/BaseModelSqlv2'; +import { Base, Model } from '../../models'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { getViewAndModelByAliasOrId, PathParams } from './helpers'; + +type BulkOperation = + | 'bulkInsert' + | 'bulkUpdate' + | 'bulkUpdateAll' + | 'bulkDelete' + | 'bulkDeleteAll'; + +export async function getModelViewBase(param: PathParams) { + const { model, view } = await getViewAndModelByAliasOrId(param); + + const base = await Base.get(model.base_id); + return { model, view, base }; +} + +export async function executeBulkOperation( + param: PathParams & { + operation: T; + options: Parameters; + } +) { + const { model, view, base } = await getModelViewBase(param); + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + return await baseModel[param.operation].apply(null, param.options); +} + +// todo: Integrate with filterArrJson bulkDataUpdateAll +export async function bulkDataInsert( + param: PathParams & { + body: any; + cookie: any; + } +) { + return await executeBulkOperation({ + ...param, + operation: 'bulkInsert', + options: [param.body, { cookie: param.cookie }], + }); +} + +// todo: Integrate with filterArrJson bulkDataUpdateAll +export async function bulkDataUpdate( + param: PathParams & { + body: any; + cookie: any; + } +) { + return await executeBulkOperation({ + ...param, + operation: 'bulkUpdate', + options: [param.body, { cookie: param.cookie }], + }); +} + +// todo: Integrate with filterArrJson bulkDataUpdateAll +export async function bulkDataUpdateAll( + param: PathParams & { + body: any; + cookie: any; + query: any; + } +) { + return await executeBulkOperation({ + ...param, + operation: 'bulkUpdateAll', + options: [param.query, param.body, { cookie: param.cookie }], + }); +} + +export async function bulkDataDelete( + param: PathParams & { + body: any; + cookie: any; + } +) { + return await executeBulkOperation({ + ...param, + operation: 'bulkDelete', + options: [param.body, { cookie: param.cookie }], + }); +} + +// todo: Integrate with filterArrJson bulkDataDeleteAll +export async function bulkDataDeleteAll( + param: PathParams & { + query: any; + } +) { + return await executeBulkOperation({ + ...param, + operation: 'bulkDeleteAll', + options: [param.query], + }); +} diff --git a/packages/nocodb/src/lib/services/dataService/helpers.ts b/packages/nocodb/src/lib/services/dataService/helpers.ts new file mode 100644 index 0000000000..bed4f78f3a --- /dev/null +++ b/packages/nocodb/src/lib/services/dataService/helpers.ts @@ -0,0 +1,31 @@ +import { NcError } from '../../meta/helpers/catchError' +import { Model, View } from '../../models' +import Project from '../../models/Project' + + +export interface PathParams { + projectName: string; + tableName: string; + viewName?: string; +} + +export async function getViewAndModelByAliasOrId(param: { + projectName: string; + tableName: string; + viewName?: string; +}) { + const project = await Project.getWithInfoByTitleOrId(param.projectName); + + const model = await Model.getByAliasOrId({ + project_id: project.id, + aliasOrId: param.tableName, + }); + const view = + param.viewName && + (await View.getByTitleOrId({ + titleOrId: param.viewName, + fk_model_id: model.id, + })); + if (!model) NcError.notFound('Table not found'); + return { model, view }; +} diff --git a/packages/nocodb/src/lib/services/dataService/index.ts b/packages/nocodb/src/lib/services/dataService/index.ts index 42d7a05556..0dd7ba3d02 100644 --- a/packages/nocodb/src/lib/services/dataService/index.ts +++ b/packages/nocodb/src/lib/services/dataService/index.ts @@ -3,14 +3,8 @@ import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; import { NcError } from '../../meta/helpers/catchError'; import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; import { Base, Model, View } from '../../models'; -import Project from '../../models/Project'; import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; - -interface PathParams { - projectName: string; - tableName: string; - viewName: string; -} +import { getViewAndModelByAliasOrId, PathParams } from './helpers' export async function dataList(param: PathParams & { query: any }) { const { model, view } = await getViewAndModelByAliasOrId(param); @@ -328,26 +322,6 @@ export async function getGroupedDataList(param: { return data; } -export async function getViewAndModelByAliasOrId(param: { - projectName: string; - tableName: string; - viewName?: string; -}) { - const project = await Project.getWithInfoByTitleOrId(param.projectName); - - const model = await Model.getByAliasOrId({ - project_id: project.id, - aliasOrId: param.tableName, - }); - const view = - param.viewName && - (await View.getByTitleOrId({ - titleOrId: param.viewName, - fk_model_id: model.id, - })); - if (!model) NcError.notFound('Table not found'); - return { model, view }; -} export async function dataListByViewId(param: { viewId: string; query: any }) { const view = await View.get(param.viewId); diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 6bcf4667aa..4a5e9dacae 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -27,3 +27,4 @@ export * as projectUserService from './projectUserService'; export * as attachmentService from './attachmentService'; export * as hookFilterService from './hookFilterService'; export * as dataService from './dataService'; +export * as bulkDataService from './dataService/bulkData'; From d4d8e8a9cd9b2b1a5a8f8a70b048c57a06434975 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 00:23:01 +0530 Subject: [PATCH 17/64] refactor: audit, cache handler and data export api(WIP) Signed-off-by: Pranav C --- .../src/lib/controllers/auditController.ts | 22 +- .../src/lib/controllers/cacheController.ts | 6 +- .../src/lib/controllers/dataApis/helpers.ts | 3 +- .../lib/controllers/dataController/export.ts | 75 +++++ .../src/lib/controllers/tableController.ts | 2 +- .../nocodb/src/lib/services/auditService.ts | 67 ++++ .../nocodb/src/lib/services/cacheService.ts | 10 + .../src/lib/services/dataService/export.ts | 54 ++++ .../src/lib/services/dataService/helpers.ts | 294 ++++++++++++++++++ .../src/lib/services/dataService/index.ts | 2 +- packages/nocodb/src/lib/services/index.ts | 2 + 11 files changed, 516 insertions(+), 21 deletions(-) create mode 100644 packages/nocodb/src/lib/controllers/dataController/export.ts create mode 100644 packages/nocodb/src/lib/services/auditService.ts create mode 100644 packages/nocodb/src/lib/services/cacheService.ts create mode 100644 packages/nocodb/src/lib/services/dataService/export.ts diff --git a/packages/nocodb/src/lib/controllers/auditController.ts b/packages/nocodb/src/lib/controllers/auditController.ts index 96b0acddee..b5d473cb60 100644 --- a/packages/nocodb/src/lib/controllers/auditController.ts +++ b/packages/nocodb/src/lib/controllers/auditController.ts @@ -7,33 +7,25 @@ import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import DOMPurify from 'isomorphic-dompurify'; import { getAjvValidatorMw } from '../meta/api/helpers'; +import { auditService } from '../services'; export async function commentRow(req: Request, res) { res.json( await Audit.insert({ ...req.body, - user: (req as any).user?.email, + user: (req as any).user, op_type: AuditOperationTypes.COMMENT, }) ); } export async function auditRowUpdate(req: Request, res) { - const model = await Model.getByIdOrName({ id: req.body.fk_model_id }); res.json( - await Audit.insert({ - fk_model_id: req.body.fk_model_id, - row_id: req.params.rowId, - op_type: AuditOperationTypes.DATA, - op_sub_type: AuditOperationSubTypes.UPDATE, - description: DOMPurify.sanitize( - `Table ${model.table_name} : field ${req.body.column_name} got changed from ${req.body.prev_value} to ${req.body.value}` - ), - details: DOMPurify.sanitize(`${req.body.column_name} - : ${req.body.prev_value} - ${req.body.value}`), - ip: (req as any).clientIp, - user: (req as any).user?.email, + await auditService.auditRowUpdate({ + rowId: req.params.rowId, + body: { + ...req.body, + }, }) ); } diff --git a/packages/nocodb/src/lib/controllers/cacheController.ts b/packages/nocodb/src/lib/controllers/cacheController.ts index 9e1fab6777..751c26178c 100644 --- a/packages/nocodb/src/lib/controllers/cacheController.ts +++ b/packages/nocodb/src/lib/controllers/cacheController.ts @@ -1,9 +1,9 @@ import catchError from '../meta/helpers/catchError'; -import NocoCache from '../cache/NocoCache'; import { Router } from 'express'; +import { cacheService } from '../services'; export async function cacheGet(_, res) { - const data = await NocoCache.export(); + const data = await cacheService.cacheGet(); res.set({ 'Content-Type': 'application/json', 'Content-Disposition': `attachment; filename="cache-export.json"`, @@ -12,7 +12,7 @@ export async function cacheGet(_, res) { } export async function cacheDelete(_, res) { - return res.json(await NocoCache.destroy()); + return res.json(await cacheService.cacheDelete()); } const router = Router(); diff --git a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts index a2b94737f2..ed48507e0d 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts @@ -36,7 +36,8 @@ export async function getViewAndModelFromRequestByAliasOrId( return { model, view }; } -export async function extractXlsxData(view: View, req: Request) { +export async function extractXlsxData(param: {view: View, query:any}) { + const { view, query } = param; const base = await Base.get(view.base_id); await view.getModelWithInfo(); diff --git a/packages/nocodb/src/lib/controllers/dataController/export.ts b/packages/nocodb/src/lib/controllers/dataController/export.ts new file mode 100644 index 0000000000..8c6cf38bea --- /dev/null +++ b/packages/nocodb/src/lib/controllers/dataController/export.ts @@ -0,0 +1,75 @@ +import { Request, Response, Router } from 'express'; +import * as XLSX from 'xlsx'; +import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import { + extractCsvData, + extractXlsxData, + getViewAndModelFromRequestByAliasOrId, +} from './helpers'; +import apiMetrics from '../../helpers/apiMetrics'; +import View from '../../../models/View'; + +async function excelDataExport(req: Request, res: Response) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + let targetView = view; + if (!targetView) { + targetView = await View.getDefaultView(model.id); + } + const { offset, elapsed, data } = await extractXlsxData(targetView, req); + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, data, targetView.title); + const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + targetView.title + )}-export.xlsx"`, + }); + res.end(buf); +} + +async function csvDataExport(req: Request, res: Response) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + let targetView = view; + if (!targetView) { + targetView = await View.getDefaultView(model.id); + } + const { offset, elapsed, data } = await extractCsvData(targetView, req); + + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + targetView.title + )}-export.csv"`, + }); + res.send(data); +} + +const router = Router({ mergeParams: true }); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/export/csv', + apiMetrics, + ncMetaAclMw(csvDataExport, 'exportCsv') +); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/csv', + apiMetrics, + ncMetaAclMw(csvDataExport, 'exportCsv') +); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/export/excel', + apiMetrics, + ncMetaAclMw(excelDataExport, 'exportExcel') +); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/excel', + apiMetrics, + ncMetaAclMw(excelDataExport, 'exportExcel') +); + +export default router; diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index 728cc1816a..8ee37c019f 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -22,7 +22,7 @@ export async function tableList(req: Request, res: Response) { } export async function tableCreate(req: Request, res) { - const result = tableService.createTable({ + const result = tableService.tableCreate({ projectId: req.params.projectId, baseId: req.params.baseId, table: req.body, diff --git a/packages/nocodb/src/lib/services/auditService.ts b/packages/nocodb/src/lib/services/auditService.ts new file mode 100644 index 0000000000..211c8d4f2d --- /dev/null +++ b/packages/nocodb/src/lib/services/auditService.ts @@ -0,0 +1,67 @@ +import { + AuditRowUpdatePayloadType, + CommentRowPayloadType, +} from 'nocodb-sdk/build/main/lib/CustomAPI'; +import Audit from '../models/Audit'; +import { AuditOperationSubTypes, AuditOperationTypes } from 'nocodb-sdk'; +import Model from '../models/Model'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; + +import DOMPurify from 'isomorphic-dompurify'; + +export async function commentRow(param: { + rowId: string; + body: CommentRowPayloadType; + user: any; +}) { + return await Audit.insert({ + ...param.body, + user: param.user?.email, + op_type: AuditOperationTypes.COMMENT, + }); +} + +export async function auditRowUpdate(param: { + rowId: string; + body: AuditRowUpdatePayloadType; +}) { + const model = await Model.getByIdOrName({ id: param.body.fk_model_id }); + return await Audit.insert({ + fk_model_id: param.body.fk_model_id, + row_id: param.rowId, + op_type: AuditOperationTypes.DATA, + op_sub_type: AuditOperationSubTypes.UPDATE, + description: DOMPurify.sanitize( + `Table ${model.table_name} : field ${param.body.column_name} got changed from ${param.body.prev_value} to ${param.body.value}` + ), + details: DOMPurify.sanitize(`${param.body.column_name} + : ${param.body.prev_value} + ${param.body.value}`), + ip: (param as any).clientIp, + user: (param as any).user?.email, + }); +} + +export async function commentList(param: { query: any }) { + return await Audit.commentsList(param.query); +} + +export async function auditList(param: { query: any; projectId: string }) { + return new PagedResponseImpl( + await Audit.projectAuditList(param.projectId, param.query), + { + count: await Audit.projectAuditCount(param.projectId), + ...param.query, + } + ); +} + +export async function commentsCount(param: { + fk_model_id: string; + ids: string[]; +}) { + return await Audit.commentsCount({ + fk_model_id: param.fk_model_id as string, + ids: param.ids as string[], + }); +} diff --git a/packages/nocodb/src/lib/services/cacheService.ts b/packages/nocodb/src/lib/services/cacheService.ts new file mode 100644 index 0000000000..6e3b83df86 --- /dev/null +++ b/packages/nocodb/src/lib/services/cacheService.ts @@ -0,0 +1,10 @@ +import NocoCache from '../cache/NocoCache'; + +export async function cacheGet() { + return await NocoCache.export(); +} + +export async function cacheDelete() { + await NocoCache.destroy() + return true +} diff --git a/packages/nocodb/src/lib/services/dataService/export.ts b/packages/nocodb/src/lib/services/dataService/export.ts new file mode 100644 index 0000000000..2ebc209262 --- /dev/null +++ b/packages/nocodb/src/lib/services/dataService/export.ts @@ -0,0 +1,54 @@ +import { Request, Response, Router } from 'express'; +import { isSystemColumn } from 'nocodb-sdk' +import * as XLSX from 'xlsx'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst' +import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import { extractXlsxData, serializeCellValue } from '../../meta/api/dataApis/helpers' +import { + extractCsvData, getViewAndModelByAliasOrId, + getViewAndModelFromRequestByAliasOrId, PathParams, +} from './helpers' +import apiMetrics from '../../helpers/apiMetrics'; +import View from '../../../models/View'; + +async function excelDataExport(param:PathParams&{ + query: any; +}) { + const { model, view } = await getViewAndModelByAliasOrId(param); + let targetView = view; + if (!targetView) { + targetView = await View.getDefaultView(model.id); + } + const { offset, elapsed, data } = await extractXlsxData({view: targetView, query:req.query }); + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, data, targetView.title); + const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + targetView.title + )}-export.xlsx"`, + }); + res.end(buf); +} + +async function csvDataExport(req: Request, res: Response) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + let targetView = view; + if (!targetView) { + targetView = await View.getDefaultView(model.id); + } + const { offset, elapsed, data } = await extractCsvData(targetView, req); + + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + targetView.title + )}-export.csv"`, + }); + res.send(data); +} diff --git a/packages/nocodb/src/lib/services/dataService/helpers.ts b/packages/nocodb/src/lib/services/dataService/helpers.ts index bed4f78f3a..02c56c7a39 100644 --- a/packages/nocodb/src/lib/services/dataService/helpers.ts +++ b/packages/nocodb/src/lib/services/dataService/helpers.ts @@ -1,6 +1,17 @@ +import { Request } from 'express' +import { nocoExecute } from 'nc-help' +import { isSystemColumn, UITypes } from 'nocodb-sdk' +import * as XLSX from 'xlsx' +import { BaseModelSqlv2 } from '../../db/sql-data-mapper/lib/sql/BaseModelSqlv2' +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst' import { NcError } from '../../meta/helpers/catchError' import { Model, View } from '../../models' +import Base from '../../models/Base' +import Column from '../../models/Column' +import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn' +import LookupColumn from '../../models/LookupColumn' import Project from '../../models/Project' +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2' export interface PathParams { @@ -29,3 +40,286 @@ export async function getViewAndModelByAliasOrId(param: { if (!model) NcError.notFound('Table not found'); return { model, view }; } + + + +export async function extractXlsxData(view: View, req: Request) { + const base = await Base.get(view.base_id); + + await view.getModelWithInfo(); + await view.getColumns(); + + view.model.columns = view.columns + .filter((c) => c.show) + .map( + (c) => + new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any) + ) + .filter((column) => !isSystemColumn(column) || view.show_system_fields); + + const baseModel = await Model.getBaseModelSQL({ + id: view.model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req); + + const fields = req.query.fields as string[]; + + const data = XLSX.utils.json_to_sheet(dbRows, { header: fields }); + + return { offset, dbRows, elapsed, data }; +} + +export async function extractCsvData(view: View, req: Request) { + const base = await Base.get(view.base_id); + const fields = req.query.fields; + + await view.getModelWithInfo(); + await view.getColumns(); + + view.model.columns = view.columns + .filter((c) => c.show) + .map( + (c) => + new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any) + ) + .filter((column) => !isSystemColumn(column) || view.show_system_fields); + + const baseModel = await Model.getBaseModelSQL({ + id: view.model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req); + + const data = papaparse.unparse( + { + fields: view.model.columns + .sort((c1, c2) => + Array.isArray(fields) + ? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any) + : 0 + ) + .filter( + (c) => + !fields || !Array.isArray(fields) || fields.includes(c.title as any) + ) + .map((c) => c.title), + data: dbRows, + }, + { + escapeFormulae: true, + } + ); + + return { offset, dbRows, elapsed, data }; +} + + +async function getDbRows(baseModel, view: View, req: Request) { + let offset = +req.query.offset || 0; + const limit = 100; + // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; + const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; + const dbRows = []; + const startTime = process.hrtime(); + let elapsed, temp; + + const listArgs: any = { ...req.query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + + for ( + elapsed = 0; + elapsed < timeout; + offset += limit, + temp = process.hrtime(startTime), + elapsed = temp[0] * 1000 + temp[1] / 1000000 + ) { + const rows = await nocoExecute( + await getAst({ + query: req.query, + includePkByDefault: false, + model: view.model, + view, + }), + await baseModel.list({ ...listArgs, offset, limit }), + {}, + req.query + ); + + if (!rows?.length) { + offset = -1; + break; + } + + for (const row of rows) { + const dbRow = { ...row }; + + for (const column of view.model.columns) { + if (isSystemColumn(column) && !view.show_system_fields) continue; + dbRow[column.title] = await serializeCellValue({ + value: row[column.title], + column, + siteUrl: req['ncSiteUrl'], + }); + } + dbRows.push(dbRow); + } + } + return { offset, dbRows, elapsed }; +} + +export async function serializeCellValue({ + value, + column, + siteUrl, + }: { + column?: Column; + value: any; + siteUrl: string; +}) { + if (!column) { + return value; + } + + if (!value) return value; + + switch (column?.uidt) { + case UITypes.Attachment: { + let data = value; + try { + if (typeof value === 'string') { + data = JSON.parse(value); + } + } catch {} + + return (data || []).map( + (attachment) => + `${encodeURI(attachment.title)}(${encodeURI( + attachment.path ? `${siteUrl}/${attachment.path}` : attachment.url + )})` + ); + } + case UITypes.Lookup: + { + const colOptions = await column.getColOptions(); + const lookupColumn = await colOptions.getLookupColumn(); + return ( + await Promise.all( + [...(Array.isArray(value) ? value : [value])].map(async (v) => + serializeCellValue({ + value: v, + column: lookupColumn, + siteUrl, + }) + ) + ) + ).join(', '); + } + break; + case UITypes.LinkToAnotherRecord: + { + const colOptions = + await column.getColOptions(); + const relatedModel = await colOptions.getRelatedTable(); + await relatedModel.getColumns(); + return [...(Array.isArray(value) ? value : [value])] + .map((v) => { + return v[relatedModel.displayValue?.title]; + }) + .join(', '); + } + break; + default: + if (value && typeof value === 'object') { + return JSON.stringify(value); + } + return value; + } +} + +export async function getColumnByIdOrName( + columnNameOrId: string, + model: Model +) { + const column = (await model.getColumns()).find( + (c) => + c.title === columnNameOrId || + c.id === columnNameOrId || + c.column_name === columnNameOrId + ); + + if (!column) + NcError.notFound(`Column with id/name '${columnNameOrId}' is not found`); + + return column; +} + + + +async function getDbRows(param: { baseModel:BaseModelSqlv2, view: View, query: any; siteUrl: string; }) { + const { baseModel, view, query = {}, siteUrl } = param; + let offset = +query.offset || 0; + const limit = 100; + // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; + const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; + const dbRows = []; + const startTime = process.hrtime(); + let elapsed, temp; + + const listArgs: any = { ...query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + + for ( + elapsed = 0; + elapsed < timeout; + offset += limit, + temp = process.hrtime(startTime), + elapsed = temp[0] * 1000 + temp[1] / 1000000 + ) { + const rows = await nocoExecute( + await getAst({ + query: query, + includePkByDefault: false, + model: view.model, + view, + }), + await baseModel.list({ ...listArgs, offset, limit }), + {}, + query + ); + + if (!rows?.length) { + offset = -1; + break; + } + + for (const row of rows) { + const dbRow = { ...row }; + + for (const column of view.model.columns) { + if (isSystemColumn(column) && !view.show_system_fields) continue; + dbRow[column.title] = await serializeCellValue({ + value: row[column.title], + column, + siteUrl, + }); + } + dbRows.push(dbRow); + } + } + return { offset, dbRows, elapsed }; +} diff --git a/packages/nocodb/src/lib/services/dataService/index.ts b/packages/nocodb/src/lib/services/dataService/index.ts index 0dd7ba3d02..668da2a598 100644 --- a/packages/nocodb/src/lib/services/dataService/index.ts +++ b/packages/nocodb/src/lib/services/dataService/index.ts @@ -1,4 +1,4 @@ -import { nocoExecute } from 'nc-help/dist/module/NocoExecute'; +import { nocoExecute } from 'nc-help'; import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; import { NcError } from '../../meta/helpers/catchError'; import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 4a5e9dacae..a9e46f8caa 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -28,3 +28,5 @@ export * as attachmentService from './attachmentService'; export * as hookFilterService from './hookFilterService'; export * as dataService from './dataService'; export * as bulkDataService from './dataService/bulkData'; +export * as cacheService from './cacheService'; +export * as auditService from './auditService'; From 675a95b132bf54cd00132334f5f962b5acf09a27 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 10:07:18 +0530 Subject: [PATCH 18/64] refactor: swagger api Signed-off-by: Pranav C --- .../src/lib/controllers/auditController.ts | 4 +- .../controllers/swagger/helpers/getPaths.ts | 48 ---- .../controllers/swagger/helpers/getSchemas.ts | 46 ---- .../swagger/helpers/getSwaggerColumnMetas.ts | 68 ------ .../swagger/helpers/getSwaggerJSON.ts | 68 ------ .../swagger/helpers/templates/headers.ts | 10 - .../swagger/helpers/templates/params.ts | 217 ------------------ .../controllers/swaggerController/index.ts | 36 +++ .../swaggerController/redocHtml.ts | 93 ++++++++ .../swaggerController/swaggerHtml.ts | 87 +++++++ packages/nocodb/src/lib/services/index.ts | 1 + .../lib/services/swaggerService/getPaths.ts | 48 ++++ .../lib/services/swaggerService/getSchemas.ts | 46 ++++ .../swaggerService/getSwaggerColumnMetas.ts | 68 ++++++ .../services/swaggerService/getSwaggerJSON.ts | 66 ++++++ .../src/lib/services/swaggerService/index.ts | 34 +++ .../swaggerService}/swagger-base.json | 0 .../swaggerService/templates/headers.ts | 10 + .../swaggerService/templates/params.ts | 217 ++++++++++++++++++ .../swaggerService}/templates/paths.ts | 0 .../swaggerService}/templates/schemas.ts | 0 21 files changed, 707 insertions(+), 460 deletions(-) create mode 100644 packages/nocodb/src/lib/controllers/swaggerController/index.ts create mode 100644 packages/nocodb/src/lib/controllers/swaggerController/redocHtml.ts create mode 100644 packages/nocodb/src/lib/controllers/swaggerController/swaggerHtml.ts create mode 100644 packages/nocodb/src/lib/services/swaggerService/getPaths.ts create mode 100644 packages/nocodb/src/lib/services/swaggerService/getSchemas.ts create mode 100644 packages/nocodb/src/lib/services/swaggerService/getSwaggerColumnMetas.ts create mode 100644 packages/nocodb/src/lib/services/swaggerService/getSwaggerJSON.ts create mode 100644 packages/nocodb/src/lib/services/swaggerService/index.ts rename packages/nocodb/src/lib/{controllers/swagger/helpers => services/swaggerService}/swagger-base.json (100%) create mode 100644 packages/nocodb/src/lib/services/swaggerService/templates/headers.ts create mode 100644 packages/nocodb/src/lib/services/swaggerService/templates/params.ts rename packages/nocodb/src/lib/{controllers/swagger/helpers => services/swaggerService}/templates/paths.ts (100%) rename packages/nocodb/src/lib/{controllers/swagger/helpers => services/swaggerService}/templates/schemas.ts (100%) diff --git a/packages/nocodb/src/lib/controllers/auditController.ts b/packages/nocodb/src/lib/controllers/auditController.ts index b5d473cb60..ee015f638a 100644 --- a/packages/nocodb/src/lib/controllers/auditController.ts +++ b/packages/nocodb/src/lib/controllers/auditController.ts @@ -1,11 +1,9 @@ import { Request, Response, Router } from 'express'; import Audit from '../models/Audit'; -import { AuditOperationSubTypes, AuditOperationTypes } from 'nocodb-sdk'; -import Model from '../models/Model'; +import { AuditOperationTypes } from 'nocodb-sdk'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import DOMPurify from 'isomorphic-dompurify'; import { getAjvValidatorMw } from '../meta/api/helpers'; import { auditService } from '../services'; diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts index 7576e52a2d..e69de29bb2 100644 --- a/packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts @@ -1,48 +0,0 @@ -import Noco from '../../../Noco'; -import Model from '../../../models/Model'; -import Project from '../../../models/Project'; -import { getModelPaths, getViewPaths } from './templates/paths'; -import { SwaggerColumn } from './getSwaggerColumnMetas'; -import { SwaggerView } from './getSwaggerJSON'; - -export default async function getPaths( - { - project, - model, - columns, - views, - }: { - project: Project; - model: Model; - columns: SwaggerColumn[]; - views: SwaggerView[]; - }, - _ncMeta = Noco.ncMeta -) { - const swaggerPaths = await getModelPaths({ - tableName: model.title, - type: model.type, - orgs: 'v1', - columns, - projectName: project.title, - }); - - for (const { view, columns: viewColumns } of views) { - const swaggerColumns = columns.filter( - (c) => viewColumns.find((vc) => vc.fk_column_id === c.column.id)?.show - ); - Object.assign( - swaggerPaths, - await getViewPaths({ - tableName: model.title, - viewName: view.title, - type: model.type, - orgs: 'v1', - columns: swaggerColumns, - projectName: project.title, - }) - ); - } - - return swaggerPaths; -} diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts index 78dc05abbb..e69de29bb2 100644 --- a/packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts @@ -1,46 +0,0 @@ -import Noco from '../../../Noco'; -import Model from '../../../models/Model'; -import Project from '../../../models/Project'; -import { getModelSchemas, getViewSchemas } from './templates/schemas'; -import { SwaggerColumn } from './getSwaggerColumnMetas'; -import { SwaggerView } from './getSwaggerJSON'; - -export default async function getSchemas( - { - project, - model, - columns, - views, - }: { - project: Project; - model: Model; - columns: SwaggerColumn[]; - views: SwaggerView[]; - }, - _ncMeta = Noco.ncMeta -) { - const swaggerSchemas = getModelSchemas({ - tableName: model.title, - orgs: 'v1', - projectName: project.title, - columns, - }); - - for (const { view, columns: viewColumns } of views) { - const swaggerColumns = columns.filter( - (c) => viewColumns.find((vc) => vc.fk_column_id === c.column.id)?.show - ); - Object.assign( - swaggerSchemas, - getViewSchemas({ - tableName: model.title, - viewName: view.title, - orgs: 'v1', - columns: swaggerColumns, - projectName: project.title, - }) - ); - } - - return swaggerSchemas; -} diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts index 46d2f202ac..e69de29bb2 100644 --- a/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts @@ -1,68 +0,0 @@ -import { UITypes } from 'nocodb-sdk'; -import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; -import SwaggerTypes from '../../../db/sql-mgr/code/routers/xc-ts/SwaggerTypes'; -import Column from '../../../models/Column'; -import Noco from '../../../Noco'; -import Project from '../../../models/Project'; - -export default async ( - columns: Column[], - project: Project, - ncMeta = Noco.ncMeta -): Promise => { - const dbType = await project.getBases().then((b) => b?.[0]?.type); - return Promise.all( - columns.map(async (c) => { - const field: SwaggerColumn = { - title: c.title, - type: 'object', - virtual: true, - column: c, - }; - - switch (c.uidt) { - case UITypes.LinkToAnotherRecord: - { - const colOpt = await c.getColOptions( - ncMeta - ); - if (colOpt) { - const relTable = await colOpt.getRelatedTable(ncMeta); - field.type = undefined; - field.$ref = `#/components/schemas/${relTable.title}Request`; - } - } - break; - case UITypes.Formula: - case UITypes.Lookup: - field.type = 'object'; - break; - case UITypes.Rollup: - field.type = 'number'; - break; - case UITypes.Attachment: - field.type = 'array'; - field.items = { - $ref: `#/components/schemas/Attachment`, - }; - break; - default: - field.virtual = false; - SwaggerTypes.setSwaggerType(c, field, dbType); - break; - } - - return field; - }) - ); -}; - -export interface SwaggerColumn { - type: any; - title: string; - description?: string; - virtual?: boolean; - $ref?: any; - column: Column; - items?: any; -} diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts index 4a17db4504..e69de29bb2 100644 --- a/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts @@ -1,68 +0,0 @@ -import FormViewColumn from '../../../models/FormViewColumn'; -import GalleryViewColumn from '../../../models/GalleryViewColumn'; -import Noco from '../../../Noco'; -import Model from '../../../models/Model'; -import swaggerBase from './swagger-base.json'; -import getPaths from './getPaths'; -import getSchemas from './getSchemas'; -import Project from '../../../models/Project'; -import getSwaggerColumnMetas from './getSwaggerColumnMetas'; -import { ViewTypes } from 'nocodb-sdk'; -import GridViewColumn from '../../../models/GridViewColumn'; -import View from '../../../models/View'; - -export default async function getSwaggerJSON( - project: Project, - models: Model[], - ncMeta = Noco.ncMeta -) { - // base swagger object - const swaggerObj = { - ...swaggerBase, - paths: {}, - components: { - ...swaggerBase.components, - schemas: { ...swaggerBase.components.schemas }, - }, - }; - - // iterate and populate swagger schema and path for models and views - for (const model of models) { - let paths = {}; - - const columns = await getSwaggerColumnMetas( - await model.getColumns(ncMeta), - project, - ncMeta - ); - - const views: SwaggerView[] = []; - - for (const view of (await model.getViews(false, ncMeta)) || []) { - if (view.type !== ViewTypes.GRID) continue; - views.push({ - view, - columns: await view.getColumns(ncMeta), - }); - } - - // skip mm tables - if (!model.mm) - paths = await getPaths({ project, model, columns, views }, ncMeta); - - const schemas = await getSchemas( - { project, model, columns, views }, - ncMeta - ); - - Object.assign(swaggerObj.paths, paths); - Object.assign(swaggerObj.components.schemas, schemas); - } - - return swaggerObj; -} - -export interface SwaggerView { - view: View; - columns: Array; -} diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts index ddc1707d41..e69de29bb2 100644 --- a/packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts @@ -1,10 +0,0 @@ -export const csvExportResponseHeader = { - 'nc-export-offset': { - schema: { - type: 'integer', - }, - description: - 'Offset of next set of data which will be helpful if there is large amount of data. It will returns `-1` if all set of data exported.', - example: '1000', - }, -}; diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts index e9c43901a5..e69de29bb2 100644 --- a/packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts +++ b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts @@ -1,217 +0,0 @@ -import { SwaggerColumn } from '../getSwaggerColumnMetas'; -import { RelationTypes, UITypes } from 'nocodb-sdk'; -import LinkToAnotherRecordColumn from '../../../../models/LinkToAnotherRecordColumn'; - -export const rowIdParam = { - schema: { - type: 'string', - }, - name: 'rowId', - in: 'path', - required: true, - example: 1, - description: - 'Primary key of the record you want to read. If the table have composite primary key then combine them by using `___` and pass it as primary key.', -}; - -export const relationTypeParam = { - schema: { - type: 'string', - enum: ['mm', 'hm'], - }, - name: 'relationType', - in: 'path', - required: true, -}; - -export const fieldsParam = { - schema: { - type: 'string', - }, - in: 'query', - name: 'fields', - description: - 'Array of field names or comma separated filed names to include in the response objects. In array syntax pass it like `fields[]=field1&fields[]=field2` or alternately `fields=field1,field2`.', -}; -export const sortParam = { - schema: { - type: 'string', - }, - in: 'query', - name: 'sort', - description: - 'Comma separated field names to sort rows, rows will sort in ascending order based on provided columns. To sort in descending order provide `-` prefix along with column name, like `-field`. Example : `sort=field1,-field2`', -}; -export const whereParam = { - schema: { - type: 'string', - }, - in: 'query', - name: 'where', - description: - 'This can be used for filtering rows, which accepts complicated where conditions. For more info visit [here](https://docs.nocodb.com/developer-resources/rest-apis#comparison-operators). Example : `where=(field1,eq,value)`', -}; -export const limitParam = { - schema: { - type: 'number', - minimum: 1, - }, - in: 'query', - name: 'limit', - description: - 'The `limit` parameter used for pagination, the response collection size depends on limit value with default value `25` and maximum value `1000`, which can be overridden by environment variables `DB_QUERY_LIMIT_DEFAULT` and `DB_QUERY_LIMIT_MAX` respectively.', - example: 25, -}; -export const offsetParam = { - schema: { - type: 'number', - minimum: 0, - }, - in: 'query', - name: 'offset', - description: - 'The `offset` parameter used for pagination, the value helps to select collection from a certain index.', - example: 0, -}; - -export const shuffleParam = { - schema: { - type: 'number', - minimum: 0, - maximum: 1, - }, - in: 'query', - name: 'shuffle', - description: - 'The `shuffle` parameter used for pagination, the response will be shuffled if it is set to 1.', - example: 0, -}; - -export const columnNameQueryParam = { - schema: { - type: 'string', - }, - in: 'query', - name: 'column_name', - description: - 'Column name of the column you want to group by, eg. `column_name=column1`', -}; - -export const columnNameParam = (columns: SwaggerColumn[]) => { - const columnNames = []; - for (const { column } of columns) { - if (column.uidt !== UITypes.LinkToAnotherRecord || column.system) continue; - columnNames.push(column.title); - } - - return { - schema: { - type: 'string', - enum: columnNames, - }, - name: 'columnName', - in: 'path', - required: true, - }; -}; - -export const referencedRowIdParam = { - schema: { - type: 'string', - }, - name: 'refRowId', - in: 'path', - required: true, -}; - -export const exportTypeParam = { - schema: { - type: 'string', - enum: ['csv', 'excel'], - }, - name: 'type', - in: 'path', - required: true, -}; - -export const csvExportOffsetParam = { - schema: { - type: 'number', - minimum: 0, - }, - in: 'query', - name: 'offset', - description: - 'Helps to start export from a certain index. You can get the next set of data offset from previous response header named `nc-export-offset`.', - example: 0, -}; - -export const nestedWhereParam = (colName) => ({ - schema: { - type: 'string', - }, - in: 'query', - name: `nested[${colName}][where]`, - description: `This can be used for filtering rows in nested column \`${colName}\`, which accepts complicated where conditions. For more info visit [here](https://docs.nocodb.com/developer-resources/rest-apis#comparison-operators). Example : \`nested[${colName}][where]=(field1,eq,value)\``, -}); - -export const nestedFieldParam = (colName) => ({ - schema: { - type: 'string', - }, - in: 'query', - name: `nested[${colName}][fields]`, - description: `Array of field names or comma separated filed names to include in the in nested column \`${colName}\` result. In array syntax pass it like \`fields[]=field1&fields[]=field2.\`. Example : \`nested[${colName}][fields]=field1,field2\``, -}); -export const nestedSortParam = (colName) => ({ - schema: { - type: 'string', - }, - in: 'query', - name: `nested[${colName}][sort]`, - description: `Comma separated field names to sort rows in nested column \`${colName}\` rows, it will sort in ascending order based on provided columns. To sort in descending order provide \`-\` prefix along with column name, like \`-field\`. Example : \`nested[${colName}][sort]=field1,-field2\``, -}); -export const nestedLimitParam = (colName) => ({ - schema: { - type: 'number', - minimum: 1, - }, - in: 'query', - name: `nested[${colName}][limit]`, - description: `The \`limit\` parameter used for pagination of nested \`${colName}\` rows, the response collection size depends on limit value and default value is \`25\`.`, - example: '25', -}); -export const nestedOffsetParam = (colName) => ({ - schema: { - type: 'number', - minimum: 0, - }, - in: 'query', - name: `nested[${colName}][offset]`, - description: `The \`offset\` parameter used for pagination of nested \`${colName}\` rows, the value helps to select collection from a certain index.`, - example: 0, -}); - -export const getNestedParams = async ( - columns: SwaggerColumn[] -): Promise => { - return await columns.reduce(async (paramsArr, { column }) => { - if (column.uidt === UITypes.LinkToAnotherRecord) { - const colOpt = await column.getColOptions(); - if (colOpt.type !== RelationTypes.BELONGS_TO) { - return [ - ...(await paramsArr), - nestedWhereParam(column.title), - nestedOffsetParam(column.title), - nestedLimitParam(column.title), - nestedFieldParam(column.title), - nestedSortParam(column.title), - ]; - } else { - return [...(await paramsArr), nestedFieldParam(column.title)]; - } - } - - return paramsArr; - }, Promise.resolve([])); -}; diff --git a/packages/nocodb/src/lib/controllers/swaggerController/index.ts b/packages/nocodb/src/lib/controllers/swaggerController/index.ts new file mode 100644 index 0000000000..1dba08f86f --- /dev/null +++ b/packages/nocodb/src/lib/controllers/swaggerController/index.ts @@ -0,0 +1,36 @@ +import { Router } from 'express' +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw' +import getSwaggerHtml from './swaggerHtml' +import getRedocHtml from './redocHtml' +import { swaggerService } from '../../services' + +async function swaggerJson(req, res) { + const swagger = await swaggerService.swaggerJson({ + projectId: req.params.projectId, + siteUrl: req.ncSiteUrl, + }) + + res.json(swagger) +} + +function swaggerHtml(_, res) { + res.send(getSwaggerHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' })) +} + +function redocHtml(_, res) { + res.send(getRedocHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' })) +} + +const router = Router({ mergeParams: true }) + +// todo: auth +router.get( + '/api/v1/db/meta/projects/:projectId/swagger.json', + ncMetaAclMw(swaggerJson, 'swaggerJson'), +) + +router.get('/api/v1/db/meta/projects/:projectId/swagger', swaggerHtml) + +router.get('/api/v1/db/meta/projects/:projectId/redoc', redocHtml) + +export default router diff --git a/packages/nocodb/src/lib/controllers/swaggerController/redocHtml.ts b/packages/nocodb/src/lib/controllers/swaggerController/redocHtml.ts new file mode 100644 index 0000000000..cc2682fdac --- /dev/null +++ b/packages/nocodb/src/lib/controllers/swaggerController/redocHtml.ts @@ -0,0 +1,93 @@ +export default ({ + ncSiteUrl, +}: { + ncSiteUrl: string; +}): string => ` + + + NocoDB API Documentation + + + + + + + + +
+ + + + +`; diff --git a/packages/nocodb/src/lib/controllers/swaggerController/swaggerHtml.ts b/packages/nocodb/src/lib/controllers/swaggerController/swaggerHtml.ts new file mode 100644 index 0000000000..27f6bef73d --- /dev/null +++ b/packages/nocodb/src/lib/controllers/swaggerController/swaggerHtml.ts @@ -0,0 +1,87 @@ +export default ({ + ncSiteUrl, +}: { + ncSiteUrl: string; +}): string => ` + + + NocoDB : API Docs + + + + + + +
+ + + +`; diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index a9e46f8caa..dbbbdacc65 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -30,3 +30,4 @@ export * as dataService from './dataService'; export * as bulkDataService from './dataService/bulkData'; export * as cacheService from './cacheService'; export * as auditService from './auditService'; +export * as swaggerService from './swaggerService'; diff --git a/packages/nocodb/src/lib/services/swaggerService/getPaths.ts b/packages/nocodb/src/lib/services/swaggerService/getPaths.ts new file mode 100644 index 0000000000..042ef515d6 --- /dev/null +++ b/packages/nocodb/src/lib/services/swaggerService/getPaths.ts @@ -0,0 +1,48 @@ +import Noco from '../../Noco'; +import Model from '../../models/Model'; +import Project from '../../models/Project'; +import { getModelPaths, getViewPaths } from './templates/paths'; +import { SwaggerColumn } from './getSwaggerColumnMetas'; +import { SwaggerView } from './getSwaggerJSON'; + +export default async function getPaths( + { + project, + model, + columns, + views, + }: { + project: Project; + model: Model; + columns: SwaggerColumn[]; + views: SwaggerView[]; + }, + _ncMeta = Noco.ncMeta +) { + const swaggerPaths = await getModelPaths({ + tableName: model.title, + type: model.type, + orgs: 'v1', + columns, + projectName: project.title, + }); + + for (const { view, columns: viewColumns } of views) { + const swaggerColumns = columns.filter( + (c) => viewColumns.find((vc) => vc.fk_column_id === c.column.id)?.show + ); + Object.assign( + swaggerPaths, + await getViewPaths({ + tableName: model.title, + viewName: view.title, + type: model.type, + orgs: 'v1', + columns: swaggerColumns, + projectName: project.title, + }) + ); + } + + return swaggerPaths; +} diff --git a/packages/nocodb/src/lib/services/swaggerService/getSchemas.ts b/packages/nocodb/src/lib/services/swaggerService/getSchemas.ts new file mode 100644 index 0000000000..5470c19fa9 --- /dev/null +++ b/packages/nocodb/src/lib/services/swaggerService/getSchemas.ts @@ -0,0 +1,46 @@ +import Noco from '../../Noco'; +import Model from '../../models/Model'; +import Project from '../../models/Project'; +import { getModelSchemas, getViewSchemas } from './templates/schemas'; +import { SwaggerColumn } from './getSwaggerColumnMetas'; +import { SwaggerView } from './getSwaggerJSON'; + +export default async function getSchemas( + { + project, + model, + columns, + views, + }: { + project: Project; + model: Model; + columns: SwaggerColumn[]; + views: SwaggerView[]; + }, + _ncMeta = Noco.ncMeta +) { + const swaggerSchemas = getModelSchemas({ + tableName: model.title, + orgs: 'v1', + projectName: project.title, + columns, + }); + + for (const { view, columns: viewColumns } of views) { + const swaggerColumns = columns.filter( + (c) => viewColumns.find((vc) => vc.fk_column_id === c.column.id)?.show + ); + Object.assign( + swaggerSchemas, + getViewSchemas({ + tableName: model.title, + viewName: view.title, + orgs: 'v1', + columns: swaggerColumns, + projectName: project.title, + }) + ); + } + + return swaggerSchemas; +} diff --git a/packages/nocodb/src/lib/services/swaggerService/getSwaggerColumnMetas.ts b/packages/nocodb/src/lib/services/swaggerService/getSwaggerColumnMetas.ts new file mode 100644 index 0000000000..45b76e4280 --- /dev/null +++ b/packages/nocodb/src/lib/services/swaggerService/getSwaggerColumnMetas.ts @@ -0,0 +1,68 @@ +import { UITypes } from 'nocodb-sdk'; +import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; +import SwaggerTypes from '../../db/sql-mgr/code/routers/xc-ts/SwaggerTypes'; +import Column from '../../models/Column'; +import Noco from '../../Noco'; +import Project from '../../models/Project'; + +export default async ( + columns: Column[], + project: Project, + ncMeta = Noco.ncMeta +): Promise => { + const dbType = await project.getBases().then((b) => b?.[0]?.type); + return Promise.all( + columns.map(async (c) => { + const field: SwaggerColumn = { + title: c.title, + type: 'object', + virtual: true, + column: c, + }; + + switch (c.uidt) { + case UITypes.LinkToAnotherRecord: + { + const colOpt = await c.getColOptions( + ncMeta + ); + if (colOpt) { + const relTable = await colOpt.getRelatedTable(ncMeta); + field.type = undefined; + field.$ref = `#/components/schemas/${relTable.title}Request`; + } + } + break; + case UITypes.Formula: + case UITypes.Lookup: + field.type = 'object'; + break; + case UITypes.Rollup: + field.type = 'number'; + break; + case UITypes.Attachment: + field.type = 'array'; + field.items = { + $ref: `#/components/schemas/Attachment`, + }; + break; + default: + field.virtual = false; + SwaggerTypes.setSwaggerType(c, field, dbType); + break; + } + + return field; + }) + ); +}; + +export interface SwaggerColumn { + type: any; + title: string; + description?: string; + virtual?: boolean; + $ref?: any; + column: Column; + items?: any; +} diff --git a/packages/nocodb/src/lib/services/swaggerService/getSwaggerJSON.ts b/packages/nocodb/src/lib/services/swaggerService/getSwaggerJSON.ts new file mode 100644 index 0000000000..9ad9cdbcf6 --- /dev/null +++ b/packages/nocodb/src/lib/services/swaggerService/getSwaggerJSON.ts @@ -0,0 +1,66 @@ +import { Model, Project, View } from '../../models' +import FormViewColumn from '../../models/FormViewColumn'; +import GalleryViewColumn from '../../models/GalleryViewColumn'; +import Noco from '../../Noco'; +import swaggerBase from './swagger-base.json'; +import getPaths from './getPaths'; +import getSchemas from './getSchemas'; +import getSwaggerColumnMetas from './getSwaggerColumnMetas'; +import { ViewTypes } from 'nocodb-sdk'; +import GridViewColumn from '../../models/GridViewColumn'; + +export default async function getSwaggerJSON( + project: Project, + models: Model[], + ncMeta = Noco.ncMeta +) { + // base swagger object + const swaggerObj = { + ...swaggerBase, + paths: {}, + components: { + ...swaggerBase.components, + schemas: { ...swaggerBase.components.schemas }, + }, + }; + + // iterate and populate swagger schema and path for models and views + for (const model of models) { + let paths = {}; + + const columns = await getSwaggerColumnMetas( + await model.getColumns(ncMeta), + project, + ncMeta + ); + + const views: SwaggerView[] = []; + + for (const view of (await model.getViews(false, ncMeta)) || []) { + if (view.type !== ViewTypes.GRID) continue; + views.push({ + view, + columns: await view.getColumns(ncMeta), + }); + } + + // skip mm tables + if (!model.mm) + paths = await getPaths({ project, model, columns, views }, ncMeta); + + const schemas = await getSchemas( + { project, model, columns, views }, + ncMeta + ); + + Object.assign(swaggerObj.paths, paths); + Object.assign(swaggerObj.components.schemas, schemas); + } + + return swaggerObj; +} + +export interface SwaggerView { + view: View; + columns: Array; +} diff --git a/packages/nocodb/src/lib/services/swaggerService/index.ts b/packages/nocodb/src/lib/services/swaggerService/index.ts new file mode 100644 index 0000000000..7ee1c073d4 --- /dev/null +++ b/packages/nocodb/src/lib/services/swaggerService/index.ts @@ -0,0 +1,34 @@ +import { NcError } from '../../meta/helpers/catchError' +import Model from '../../models/Model' +import Project from '../../models/Project' +import getSwaggerJSON from './getSwaggerJSON' + +export async function swaggerJson(param:{projectId:string; siteUrl:string}){ + const project = await Project.get(param.projectId); + + if (!project) NcError.notFound(); + + const models = await Model.list({ + project_id: param.projectId, + base_id: null, + }); + + const swagger = await getSwaggerJSON(project, models); + + swagger.servers = [ + { + url: param.siteUrl, + }, + { + url: '{customUrl}', + variables: { + customUrl: { + default: param.siteUrl, + description: 'Provide custom nocodb app base url', + }, + }, + }, + ] as any; + + return swagger; +} diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/swagger-base.json b/packages/nocodb/src/lib/services/swaggerService/swagger-base.json similarity index 100% rename from packages/nocodb/src/lib/controllers/swagger/helpers/swagger-base.json rename to packages/nocodb/src/lib/services/swaggerService/swagger-base.json diff --git a/packages/nocodb/src/lib/services/swaggerService/templates/headers.ts b/packages/nocodb/src/lib/services/swaggerService/templates/headers.ts new file mode 100644 index 0000000000..ddc1707d41 --- /dev/null +++ b/packages/nocodb/src/lib/services/swaggerService/templates/headers.ts @@ -0,0 +1,10 @@ +export const csvExportResponseHeader = { + 'nc-export-offset': { + schema: { + type: 'integer', + }, + description: + 'Offset of next set of data which will be helpful if there is large amount of data. It will returns `-1` if all set of data exported.', + example: '1000', + }, +}; diff --git a/packages/nocodb/src/lib/services/swaggerService/templates/params.ts b/packages/nocodb/src/lib/services/swaggerService/templates/params.ts new file mode 100644 index 0000000000..698dbc5eb5 --- /dev/null +++ b/packages/nocodb/src/lib/services/swaggerService/templates/params.ts @@ -0,0 +1,217 @@ +import { SwaggerColumn } from '../getSwaggerColumnMetas'; +import { RelationTypes, UITypes } from 'nocodb-sdk'; +import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; + +export const rowIdParam = { + schema: { + type: 'string', + }, + name: 'rowId', + in: 'path', + required: true, + example: 1, + description: + 'Primary key of the record you want to read. If the table have composite primary key then combine them by using `___` and pass it as primary key.', +}; + +export const relationTypeParam = { + schema: { + type: 'string', + enum: ['mm', 'hm'], + }, + name: 'relationType', + in: 'path', + required: true, +}; + +export const fieldsParam = { + schema: { + type: 'string', + }, + in: 'query', + name: 'fields', + description: + 'Array of field names or comma separated filed names to include in the response objects. In array syntax pass it like `fields[]=field1&fields[]=field2` or alternately `fields=field1,field2`.', +}; +export const sortParam = { + schema: { + type: 'string', + }, + in: 'query', + name: 'sort', + description: + 'Comma separated field names to sort rows, rows will sort in ascending order based on provided columns. To sort in descending order provide `-` prefix along with column name, like `-field`. Example : `sort=field1,-field2`', +}; +export const whereParam = { + schema: { + type: 'string', + }, + in: 'query', + name: 'where', + description: + 'This can be used for filtering rows, which accepts complicated where conditions. For more info visit [here](https://docs.nocodb.com/developer-resources/rest-apis#comparison-operators). Example : `where=(field1,eq,value)`', +}; +export const limitParam = { + schema: { + type: 'number', + minimum: 1, + }, + in: 'query', + name: 'limit', + description: + 'The `limit` parameter used for pagination, the response collection size depends on limit value with default value `25` and maximum value `1000`, which can be overridden by environment variables `DB_QUERY_LIMIT_DEFAULT` and `DB_QUERY_LIMIT_MAX` respectively.', + example: 25, +}; +export const offsetParam = { + schema: { + type: 'number', + minimum: 0, + }, + in: 'query', + name: 'offset', + description: + 'The `offset` parameter used for pagination, the value helps to select collection from a certain index.', + example: 0, +}; + +export const shuffleParam = { + schema: { + type: 'number', + minimum: 0, + maximum: 1, + }, + in: 'query', + name: 'shuffle', + description: + 'The `shuffle` parameter used for pagination, the response will be shuffled if it is set to 1.', + example: 0, +}; + +export const columnNameQueryParam = { + schema: { + type: 'string', + }, + in: 'query', + name: 'column_name', + description: + 'Column name of the column you want to group by, eg. `column_name=column1`', +}; + +export const columnNameParam = (columns: SwaggerColumn[]) => { + const columnNames = []; + for (const { column } of columns) { + if (column.uidt !== UITypes.LinkToAnotherRecord || column.system) continue; + columnNames.push(column.title); + } + + return { + schema: { + type: 'string', + enum: columnNames, + }, + name: 'columnName', + in: 'path', + required: true, + }; +}; + +export const referencedRowIdParam = { + schema: { + type: 'string', + }, + name: 'refRowId', + in: 'path', + required: true, +}; + +export const exportTypeParam = { + schema: { + type: 'string', + enum: ['csv', 'excel'], + }, + name: 'type', + in: 'path', + required: true, +}; + +export const csvExportOffsetParam = { + schema: { + type: 'number', + minimum: 0, + }, + in: 'query', + name: 'offset', + description: + 'Helps to start export from a certain index. You can get the next set of data offset from previous response header named `nc-export-offset`.', + example: 0, +}; + +export const nestedWhereParam = (colName) => ({ + schema: { + type: 'string', + }, + in: 'query', + name: `nested[${colName}][where]`, + description: `This can be used for filtering rows in nested column \`${colName}\`, which accepts complicated where conditions. For more info visit [here](https://docs.nocodb.com/developer-resources/rest-apis#comparison-operators). Example : \`nested[${colName}][where]=(field1,eq,value)\``, +}); + +export const nestedFieldParam = (colName) => ({ + schema: { + type: 'string', + }, + in: 'query', + name: `nested[${colName}][fields]`, + description: `Array of field names or comma separated filed names to include in the in nested column \`${colName}\` result. In array syntax pass it like \`fields[]=field1&fields[]=field2.\`. Example : \`nested[${colName}][fields]=field1,field2\``, +}); +export const nestedSortParam = (colName) => ({ + schema: { + type: 'string', + }, + in: 'query', + name: `nested[${colName}][sort]`, + description: `Comma separated field names to sort rows in nested column \`${colName}\` rows, it will sort in ascending order based on provided columns. To sort in descending order provide \`-\` prefix along with column name, like \`-field\`. Example : \`nested[${colName}][sort]=field1,-field2\``, +}); +export const nestedLimitParam = (colName) => ({ + schema: { + type: 'number', + minimum: 1, + }, + in: 'query', + name: `nested[${colName}][limit]`, + description: `The \`limit\` parameter used for pagination of nested \`${colName}\` rows, the response collection size depends on limit value and default value is \`25\`.`, + example: '25', +}); +export const nestedOffsetParam = (colName) => ({ + schema: { + type: 'number', + minimum: 0, + }, + in: 'query', + name: `nested[${colName}][offset]`, + description: `The \`offset\` parameter used for pagination of nested \`${colName}\` rows, the value helps to select collection from a certain index.`, + example: 0, +}); + +export const getNestedParams = async ( + columns: SwaggerColumn[] +): Promise => { + return await columns.reduce(async (paramsArr, { column }) => { + if (column.uidt === UITypes.LinkToAnotherRecord) { + const colOpt = await column.getColOptions(); + if (colOpt.type !== RelationTypes.BELONGS_TO) { + return [ + ...(await paramsArr), + nestedWhereParam(column.title), + nestedOffsetParam(column.title), + nestedLimitParam(column.title), + nestedFieldParam(column.title), + nestedSortParam(column.title), + ]; + } else { + return [...(await paramsArr), nestedFieldParam(column.title)]; + } + } + + return paramsArr; + }, Promise.resolve([])); +}; diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/templates/paths.ts b/packages/nocodb/src/lib/services/swaggerService/templates/paths.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/swagger/helpers/templates/paths.ts rename to packages/nocodb/src/lib/services/swaggerService/templates/paths.ts diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/templates/schemas.ts b/packages/nocodb/src/lib/services/swaggerService/templates/schemas.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/swagger/helpers/templates/schemas.ts rename to packages/nocodb/src/lib/services/swaggerService/templates/schemas.ts From b3a5b4802d015d45b19ad48b1e5401ddbb030be9 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 10:57:31 +0530 Subject: [PATCH 19/64] refactor: user api Signed-off-by: Pranav C --- .../lib/controllers/userController/index.ts | 1 + .../userController/initStrategies.ts | 332 +++++++++++++ .../userController/ui/auth/emailVerify.ts | 70 +++ .../userController/ui/auth/resetPassword.ts | 108 ++++ .../ui/emailTemplates/forgotPassword.ts | 171 +++++++ .../ui/emailTemplates/invite.ts | 208 ++++++++ .../ui/emailTemplates/verify.ts | 207 ++++++++ .../controllers/userController/userApis.ts | 462 ++++++++++++++++++ packages/nocodb/src/lib/services/index.ts | 1 + .../src/lib/services/userService/helpers.ts | 23 + .../src/lib/services/userService/index.ts | 301 ++++++++++++ .../services/userService/initAdminFromEnv.ts | 283 +++++++++++ 12 files changed, 2167 insertions(+) create mode 100644 packages/nocodb/src/lib/controllers/userController/index.ts create mode 100644 packages/nocodb/src/lib/controllers/userController/initStrategies.ts create mode 100644 packages/nocodb/src/lib/controllers/userController/ui/auth/emailVerify.ts create mode 100644 packages/nocodb/src/lib/controllers/userController/ui/auth/resetPassword.ts create mode 100644 packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/forgotPassword.ts create mode 100644 packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/invite.ts create mode 100644 packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/verify.ts create mode 100644 packages/nocodb/src/lib/controllers/userController/userApis.ts create mode 100644 packages/nocodb/src/lib/services/userService/helpers.ts create mode 100644 packages/nocodb/src/lib/services/userService/index.ts create mode 100644 packages/nocodb/src/lib/services/userService/initAdminFromEnv.ts diff --git a/packages/nocodb/src/lib/controllers/userController/index.ts b/packages/nocodb/src/lib/controllers/userController/index.ts new file mode 100644 index 0000000000..c4a5003430 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/userController/index.ts @@ -0,0 +1 @@ +export * from './userApis'; diff --git a/packages/nocodb/src/lib/controllers/userController/initStrategies.ts b/packages/nocodb/src/lib/controllers/userController/initStrategies.ts new file mode 100644 index 0000000000..3b6b99c9aa --- /dev/null +++ b/packages/nocodb/src/lib/controllers/userController/initStrategies.ts @@ -0,0 +1,332 @@ +import { OrgUserRoles } from 'nocodb-sdk'; +import { promisify } from 'util'; +import { Strategy as CustomStrategy } from 'passport-custom'; +import passport from 'passport'; +import passportJWT from 'passport-jwt'; +import { Strategy as AuthTokenStrategy } from 'passport-auth-token'; +import { Strategy as GoogleStrategy } from 'passport-google-oauth20'; + +const PassportLocalStrategy = require('passport-local').Strategy; +const ExtractJwt = passportJWT.ExtractJwt; +const JwtStrategy = passportJWT.Strategy; + +const jwtOptions = { + jwtFromRequest: ExtractJwt.fromHeader('xc-auth'), +}; + +import bcrypt from 'bcryptjs'; +import NocoCache from '../../cache/NocoCache'; +import { ApiToken, Plugin, Project, ProjectUser, User } from '../../models'; +import Noco from '../../Noco'; +import { CacheGetType, CacheScope } from '../../utils/globals'; +import { userService } from '../../services'; + +export function initStrategies(router): void { + passport.use( + 'authtoken', + new AuthTokenStrategy( + { headerFields: ['xc-token'], passReqToCallback: true }, + (req, token, done) => { + ApiToken.getByToken(token) + .then((apiToken) => { + if (!apiToken) { + return done({ msg: 'Invalid token' }); + } + + if (!apiToken.fk_user_id) return done(null, { roles: 'editor' }); + User.get(apiToken.fk_user_id) + .then((user) => { + user['is_api_token'] = true; + if (req.ncProjectId) { + ProjectUser.get(req.ncProjectId, user.id) + .then(async (projectUser) => { + user.roles = projectUser?.roles || user.roles; + user.roles = + user.roles === 'owner' ? 'owner,creator' : user.roles; + // + (user.roles ? `,${user.roles}` : ''); + // todo : cache + // await NocoCache.set(`${CacheScope.USER}:${key}`, user); + done(null, user); + }) + .catch((e) => done(e)); + } else { + return done(null, user); + } + }) + .catch((e) => { + console.log(e); + done({ msg: 'User not found' }); + }); + }) + .catch((e) => { + console.log(e); + done({ msg: 'Invalid token' }); + }); + } + ) + ); + + passport.serializeUser(function ( + { + id, + email, + email_verified, + roles: _roles, + provider, + firstname, + lastname, + isAuthorized, + isPublicBase, + token_version, + }, + done + ) { + const roles = (_roles || '') + .split(',') + .reduce((obj, role) => Object.assign(obj, { [role]: true }), {}); + if (roles.owner) { + roles.creator = true; + } + done(null, { + isAuthorized, + isPublicBase, + id, + email, + email_verified, + provider, + firstname, + lastname, + roles, + token_version, + }); + }); + + passport.deserializeUser(function (user, done) { + done(null, user); + }); + + passport.use( + new JwtStrategy( + { + secretOrKey: Noco.getConfig().auth.jwt.secret, + ...jwtOptions, + passReqToCallback: true, + ...Noco.getConfig().auth.jwt.options, + }, + async (req, jwtPayload, done) => { + // todo: improve this + if ( + req.ncProjectId && + jwtPayload.roles?.split(',').includes(OrgUserRoles.SUPER_ADMIN) + ) { + return User.getByEmail(jwtPayload?.email).then(async (user) => { + return done(null, { + ...user, + roles: `owner,creator,${OrgUserRoles.SUPER_ADMIN}`, + }); + }); + } + + const keyVals = [jwtPayload?.email]; + if (req.ncProjectId) { + keyVals.push(req.ncProjectId); + } + const key = keyVals.join('___'); + const cachedVal = await NocoCache.get( + `${CacheScope.USER}:${key}`, + CacheGetType.TYPE_OBJECT + ); + + if (cachedVal) { + if ( + !cachedVal.token_version || + !jwtPayload.token_version || + cachedVal.token_version !== jwtPayload.token_version + ) { + return done(new Error('Token Expired. Please login again.')); + } + return done(null, cachedVal); + } + + User.getByEmail(jwtPayload?.email) + .then(async (user) => { + if ( + !user.token_version || + !jwtPayload.token_version || + user.token_version !== jwtPayload.token_version + ) { + return done(new Error('Token Expired. Please login again.')); + } + if (req.ncProjectId) { + // this.xcMeta + // .metaGet(req.ncProjectId, null, 'nc_projects_users', { + // user_id: user?.id + // }) + + ProjectUser.get(req.ncProjectId, user.id) + .then(async (projectUser) => { + user.roles = projectUser?.roles || user.roles; + user.roles = + user.roles === 'owner' ? 'owner,creator' : user.roles; + // + (user.roles ? `,${user.roles}` : ''); + + await NocoCache.set(`${CacheScope.USER}:${key}`, user); + done(null, user); + }) + .catch((e) => done(e)); + } else { + // const roles = projectUser?.roles ? JSON.parse(projectUser.roles) : {guest: true}; + if (user) { + await NocoCache.set(`${CacheScope.USER}:${key}`, user); + return done(null, user); + } else { + return done(new Error('User not found')); + } + } + }) + .catch((err) => { + return done(err); + }); + } + ) + ); + + passport.use( + new PassportLocalStrategy( + { + usernameField: 'email', + session: false, + }, + async (email, password, done) => { + try { + const user = await User.getByEmail(email); + if (!user) { + return done({ msg: `Email ${email} is not registered!` }); + } + + if (!user.salt) { + return done({ + msg: `Please sign up with the invite token first or reset the password by clicking Forgot your password.`, + }); + } + + const hashedPassword = await promisify(bcrypt.hash)( + password, + user.salt + ); + if (user.password !== hashedPassword) { + return done({ msg: `Password not valid!` }); + } else { + return done(null, user); + } + } catch (e) { + done(e); + } + } + ) + ); + + passport.use( + 'baseView', + new CustomStrategy(async (req: any, callback) => { + let user; + if (req.headers['xc-shared-base-id']) { + // const cacheKey = `nc_shared_bases||${req.headers['xc-shared-base-id']}`; + + let sharedProject = null; + + if (!sharedProject) { + sharedProject = await Project.getByUuid( + req.headers['xc-shared-base-id'] + ); + } + user = { + roles: sharedProject?.roles, + }; + } + + callback(null, user); + }) + ); + + // mostly copied from older code + Plugin.getPluginByTitle('Google').then((googlePlugin) => { + if (googlePlugin && googlePlugin.input) { + const settings = JSON.parse(googlePlugin.input); + process.env.NC_GOOGLE_CLIENT_ID = settings.client_id; + process.env.NC_GOOGLE_CLIENT_SECRET = settings.client_secret; + } + + if ( + process.env.NC_GOOGLE_CLIENT_ID && + process.env.NC_GOOGLE_CLIENT_SECRET + ) { + const googleAuthParamsOrig = GoogleStrategy.prototype.authorizationParams; + GoogleStrategy.prototype.authorizationParams = (options: any) => { + const params = googleAuthParamsOrig.call(this, options); + + if (options.state) { + params.state = options.state; + } + + return params; + }; + + const clientConfig = { + clientID: process.env.NC_GOOGLE_CLIENT_ID, + clientSecret: process.env.NC_GOOGLE_CLIENT_SECRET, + // todo: update url + callbackURL: 'http://localhost:3000', + passReqToCallback: true, + }; + + const googleStrategy = new GoogleStrategy( + clientConfig, + async (req, _accessToken, _refreshToken, profile, done) => { + const email = profile.emails[0].value; + + User.getByEmail(email) + .then(async (user) => { + if (user) { + // if project id defined extract project level roles + if (req.ncProjectId) { + ProjectUser.get(req.ncProjectId, user.id) + .then(async (projectUser) => { + user.roles = projectUser?.roles || user.roles; + user.roles = + user.roles === 'owner' ? 'owner,creator' : user.roles; + // + (user.roles ? `,${user.roles}` : ''); + + done(null, user); + }) + .catch((e) => done(e)); + } else { + return done(null, user); + } + // if user not found create new user if allowed + // or return error + } else { + const salt = await promisify(bcrypt.genSalt)(10); + const user = await userService.registerNewUserIfAllowed({ + firstname: null, + lastname: null, + email_verification_token: null, + email: profile.emails[0].value, + password: '', + salt, + }); + return done(null, user); + } + }) + .catch((err) => { + return done(err); + }); + } + ); + + passport.use(googleStrategy); + } + }); + + router.use(passport.initialize()); +} diff --git a/packages/nocodb/src/lib/controllers/userController/ui/auth/emailVerify.ts b/packages/nocodb/src/lib/controllers/userController/ui/auth/emailVerify.ts new file mode 100644 index 0000000000..f7412b12ba --- /dev/null +++ b/packages/nocodb/src/lib/controllers/userController/ui/auth/emailVerify.ts @@ -0,0 +1,70 @@ +export default ` + + + NocoDB - Verify Email + + + + + + + +
+ + + + + + Email verified successfully! + + + {{errMsg}} + + + + + + + +
+ + + + + +`; diff --git a/packages/nocodb/src/lib/controllers/userController/ui/auth/resetPassword.ts b/packages/nocodb/src/lib/controllers/userController/ui/auth/resetPassword.ts new file mode 100644 index 0000000000..514fc6d739 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/userController/ui/auth/resetPassword.ts @@ -0,0 +1,108 @@ +export default ` + + + NocoDB - Reset Password + + + + + + + +
+ + + + + + Password reset successful! + + + + + + +
+ + + + + +`; diff --git a/packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/forgotPassword.ts b/packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/forgotPassword.ts new file mode 100644 index 0000000000..afb2f5849a --- /dev/null +++ b/packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/forgotPassword.ts @@ -0,0 +1,171 @@ +export default ` + + + + + Simple Transactional Email + + + + +
+ + + + + + + + +`; diff --git a/packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/invite.ts b/packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/invite.ts new file mode 100644 index 0000000000..fc81f9409e --- /dev/null +++ b/packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/invite.ts @@ -0,0 +1,208 @@ +export default ` + + + + + Simple Transactional Email + + + + + + + + + + + + + +`; diff --git a/packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/verify.ts b/packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/verify.ts new file mode 100644 index 0000000000..11702cc659 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/userController/ui/emailTemplates/verify.ts @@ -0,0 +1,207 @@ +export default ` + + + + + Simple Transactional Email + + + + + + + + + + + + + +`; diff --git a/packages/nocodb/src/lib/controllers/userController/userApis.ts b/packages/nocodb/src/lib/controllers/userController/userApis.ts new file mode 100644 index 0000000000..367eb4591b --- /dev/null +++ b/packages/nocodb/src/lib/controllers/userController/userApis.ts @@ -0,0 +1,462 @@ +import { Request, Response } from 'express'; +import { TableType, validatePassword } from 'nocodb-sdk'; +import { Tele } from 'nc-help'; + +const { isEmail } = require('validator'); +import * as ejs from 'ejs'; + +import bcrypt from 'bcryptjs'; +import { promisify } from 'util'; + +const { v4: uuidv4 } = require('uuid'); + +import passport from 'passport'; +import { getAjvValidatorMw } from '../../meta/api/helpers'; +import catchError, { NcError } from '../../meta/helpers/catchError'; +import extractProjectIdAndAuthenticate from '../../meta/helpers/extractProjectIdAndAuthenticate'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; +import { Audit, User } from '../../models'; +import Noco from '../../Noco'; +import { genJwt } from './helpers'; +import { userService } from '../../services'; + +export async function signup(req: Request, res: Response) { + const { + email: _email, + firstname, + lastname, + token, + ignore_subscribe, + } = req.body; + let { password } = req.body; + + // validate password and throw error if password is satisfying the conditions + const { valid, error } = validatePassword(password); + if (!valid) { + NcError.badRequest(`Password : ${error}`); + } + + if (!isEmail(_email)) { + NcError.badRequest(`Invalid email`); + } + + const email = _email.toLowerCase(); + + let user = await User.getByEmail(email); + + if (user) { + if (token) { + if (token !== user.invite_token) { + NcError.badRequest(`Invalid invite url`); + } else if (user.invite_token_expires < new Date()) { + NcError.badRequest( + 'Expired invite url, Please contact super admin to get a new invite url' + ); + } + } else { + // todo : opening up signup for timebeing + // return next(new Error(`Email '${email}' already registered`)); + } + } + + const salt = await promisify(bcrypt.genSalt)(10); + password = await promisify(bcrypt.hash)(password, salt); + const email_verification_token = uuidv4(); + + if (!ignore_subscribe) { + Tele.emit('evt_subscribe', email); + } + + if (user) { + if (token) { + await User.update(user.id, { + firstname, + lastname, + salt, + password, + email_verification_token, + invite_token: null, + invite_token_expires: null, + email: user.email, + }); + } else { + NcError.badRequest('User already exist'); + } + } else { + await userService.registerNewUserIfAllowed({ + firstname, + lastname, + email, + salt, + password, + email_verification_token, + }); + } + user = await User.getByEmail(email); + + try { + const template = (await import('./ui/emailTemplates/verify')).default; + await ( + await NcPluginMgrv2.emailAdapter() + ).mailSend({ + to: email, + subject: 'Verify email', + html: ejs.render(template, { + verifyLink: + (req as any).ncSiteUrl + + `/email/verify/${user.email_verification_token}`, + }), + }); + } catch (e) { + console.log( + 'Warning : `mailSend` failed, Please configure emailClient configuration.' + ); + } + await promisify((req as any).login.bind(req))(user); + const refreshToken = userService.randomTokenString(); + await User.update(user.id, { + refresh_token: refreshToken, + email: user.email, + }); + + setTokenCookie(res, refreshToken); + + user = (req as any).user; + + await Audit.insert({ + op_type: 'AUTHENTICATION', + op_sub_type: 'SIGNUP', + user: user.email, + description: `signed up `, + ip: (req as any).clientIp, + }); + + res.json({ + token: genJwt(user, Noco.getConfig()), + } as any); +} + +async function successfulSignIn({ + user, + err, + info, + req, + res, + auditDescription, +}) { + try { + if (!user || !user.email) { + if (err) { + return res.status(400).send(err); + } + if (info) { + return res.status(400).send(info); + } + return res.status(400).send({ msg: 'Your signin has failed' }); + } + + await promisify((req as any).login.bind(req))(user); + const refreshToken = userService.randomTokenString(); + + if (!user.token_version) { + user.token_version = userService.randomTokenString(); + } + + await User.update(user.id, { + refresh_token: refreshToken, + email: user.email, + token_version: user.token_version, + }); + setTokenCookie(res, refreshToken); + + await Audit.insert({ + op_type: 'AUTHENTICATION', + op_sub_type: 'SIGNIN', + user: user.email, + ip: req.clientIp, + description: auditDescription, + }); + + res.json({ + token: genJwt(user, Noco.getConfig()), + } as any); + } catch (e) { + console.log(e); + throw e; + } +} + +async function signin(req, res, next) { + passport.authenticate( + 'local', + { session: false }, + async (err, user, info): Promise => + await successfulSignIn({ + user, + err, + info, + req, + res, + auditDescription: 'signed in', + }) + )(req, res, next); +} + +async function googleSignin(req, res, next) { + passport.authenticate( + 'google', + { + session: false, + callbackURL: req.ncSiteUrl + Noco.getConfig().dashboardPath, + }, + async (err, user, info): Promise => + await successfulSignIn({ + user, + err, + info, + req, + res, + auditDescription: 'signed in using Google Auth', + }) + )(req, res, next); +} + +function setTokenCookie(res: Response, token): void { + // create http only cookie with refresh token that expires in 7 days + const cookieOptions = { + httpOnly: true, + expires: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000), + }; + res.cookie('refresh_token', token, cookieOptions); +} + +async function me(req, res): Promise { + res.json(req?.session?.passport?.user ?? {}); +} + +async function passwordChange(req: Request, res): Promise { + if (!(req as any).isAuthenticated()) { + NcError.forbidden('Not allowed'); + } + + await userService.passwordChange({ + user: req['user'], + req, + body: req.body, + }); + + res.json({ msg: 'Password updated successfully' }); +} + +async function passwordForgot(req: Request, res): Promise { + await userService.passwordForgot({ + siteUrl: (req as any).ncSiteUrl, + body: req.body, + req, + }); + + res.json({ msg: 'Please check your email to reset the password' }); +} + +async function tokenValidate(req, res): Promise { + await userService.tokenValidate({ + token: req.params.tokenId, + }); + res.json(true); +} + +async function passwordReset(req, res): Promise { + await userService.passwordReset({ + token: req.params.tokenId, + body: req.body, + req, + }); + + res.json({ msg: 'Password reset successful' }); +} + +async function emailVerification(req, res): Promise { + await userService.emailVerification({ + token: req.params.tokenId, + req, + }); + + res.json({ msg: 'Email verified successfully' }); +} + +async function refreshToken(req, res): Promise { + try { + if (!req?.cookies?.refresh_token) { + return res.status(400).json({ msg: 'Missing refresh token' }); + } + + const user = await User.getByRefreshToken(req.cookies.refresh_token); + + if (!user) { + return res.status(400).json({ msg: 'Invalid refresh token' }); + } + + const refreshToken = userService.randomTokenString(); + + await User.update(user.id, { + email: user.email, + refresh_token: refreshToken, + }); + + setTokenCookie(res, refreshToken); + + res.json({ + token: genJwt(user, Noco.getConfig()), + } as any); + } catch (e) { + return res.status(400).json({ msg: e.message }); + } +} + +async function renderPasswordReset(req, res): Promise { + try { + res.send( + ejs.render((await import('./ui/auth/resetPassword')).default, { + ncPublicUrl: process.env.NC_PUBLIC_URL || '', + token: JSON.stringify(req.params.tokenId), + baseUrl: `/`, + }) + ); + } catch (e) { + return res.status(400).json({ msg: e.message }); + } +} + +const mapRoutes = (router) => { + // todo: old api - /auth/signup?tool=1 + router.post( + '/auth/user/signup', + getAjvValidatorMw('swagger.json#/components/schemas/SignUpReq'), + catchError(signup) + ); + router.post( + '/auth/user/signin', + getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), + catchError(signin) + ); + router.get('/auth/user/me', extractProjectIdAndAuthenticate, catchError(me)); + router.post( + '/auth/password/forgot', + getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), + catchError(passwordForgot) + ); + router.post('/auth/token/validate/:tokenId', catchError(tokenValidate)); + router.post( + '/auth/password/reset/:tokenId', + getAjvValidatorMw('swagger.json#/components/schemas/PasswordResetReq'), + catchError(passwordReset) + ); + router.post('/auth/email/validate/:tokenId', catchError(emailVerification)); + router.post( + '/user/password/change', + getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), + ncMetaAclMw(passwordChange, 'passwordChange') + ); + router.post('/auth/token/refresh', catchError(refreshToken)); + + /* Google auth apis */ + + router.post(`/auth/google/genTokenByCode`, catchError(googleSignin)); + + router.get('/auth/google', (req: any, res, next) => + passport.authenticate('google', { + scope: ['profile', 'email'], + state: req.query.state, + callbackURL: req.ncSiteUrl + Noco.getConfig().dashboardPath, + })(req, res, next) + ); + + // deprecated APIs + router.post( + '/api/v1/db/auth/user/signup', + getAjvValidatorMw('swagger.json#/components/schemas/SignUpReq'), + catchError(signup) + ); + router.post( + '/api/v1/db/auth/user/signin', + getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), + catchError(signin) + ); + router.get( + '/api/v1/db/auth/user/me', + extractProjectIdAndAuthenticate, + catchError(me) + ); + router.post( + '/api/v1/db/auth/password/forgot', + getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), + catchError(passwordForgot) + ); + router.post( + '/api/v1/db/auth/token/validate/:tokenId', + catchError(tokenValidate) + ); + router.post( + '/api/v1/db/auth/password/reset/:tokenId', + getAjvValidatorMw('swagger.json#/components/schemas/PasswordResetReq'), + catchError(passwordReset) + ); + router.post( + '/api/v1/db/auth/email/validate/:tokenId', + catchError(emailVerification) + ); + router.post( + '/api/v1/db/auth/password/change', + getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), + ncMetaAclMw(passwordChange, 'passwordChange') + ); + router.post('/api/v1/db/auth/token/refresh', catchError(refreshToken)); + router.get( + '/api/v1/db/auth/password/reset/:tokenId', + catchError(renderPasswordReset) + ); + + // new API + router.post( + '/api/v1/auth/user/signup', + getAjvValidatorMw('swagger.json#/components/schemas/SignUpReq'), + catchError(signup) + ); + router.post( + '/api/v1/auth/user/signin', + getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), + catchError(signin) + ); + router.get( + '/api/v1/auth/user/me', + extractProjectIdAndAuthenticate, + catchError(me) + ); + router.post( + '/api/v1/auth/password/forgot', + getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), + catchError(passwordForgot) + ); + router.post( + '/api/v1/auth/token/validate/:tokenId', + catchError(tokenValidate) + ); + router.post( + '/api/v1/auth/password/reset/:tokenId', + catchError(passwordReset) + ); + router.post( + '/api/v1/auth/email/validate/:tokenId', + catchError(emailVerification) + ); + router.post( + '/api/v1/auth/password/change', + getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), + ncMetaAclMw(passwordChange, 'passwordChange') + ); + router.post('/api/v1/auth/token/refresh', catchError(refreshToken)); + // respond with password reset page + router.get('/auth/password/reset/:tokenId', catchError(renderPasswordReset)); +}; +export { mapRoutes as userApis }; diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index dbbbdacc65..d0fe29bdcf 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -31,3 +31,4 @@ export * as bulkDataService from './dataService/bulkData'; export * as cacheService from './cacheService'; export * as auditService from './auditService'; export * as swaggerService from './swaggerService'; +export * as userService from './userService'; diff --git a/packages/nocodb/src/lib/services/userService/helpers.ts b/packages/nocodb/src/lib/services/userService/helpers.ts new file mode 100644 index 0000000000..4cfe0bd687 --- /dev/null +++ b/packages/nocodb/src/lib/services/userService/helpers.ts @@ -0,0 +1,23 @@ +import * as jwt from 'jsonwebtoken'; +import crypto from 'crypto'; +import { NcConfig } from '../../../interface/config' +import { User } from '../../models' + +export function genJwt(user: User, config: NcConfig) { + return jwt.sign( + { + email: user.email, + firstname: user.firstname, + lastname: user.lastname, + id: user.id, + roles: user.roles, + token_version: user.token_version, + }, + config.auth.jwt.secret, + config.auth.jwt.options + ); +} + +export function randomTokenString(): string { + return crypto.randomBytes(40).toString('hex'); +} diff --git a/packages/nocodb/src/lib/services/userService/index.ts b/packages/nocodb/src/lib/services/userService/index.ts new file mode 100644 index 0000000000..73b9a9948c --- /dev/null +++ b/packages/nocodb/src/lib/services/userService/index.ts @@ -0,0 +1,301 @@ +import { Request, Response } from 'express'; +import { + PasswordChangeReqType, + PasswordForgotReqType, PasswordResetReqType, + SignUpReqType, + TableType, + UserType, + validatePassword, +} from 'nocodb-sdk' +import { OrgUserRoles } from 'nocodb-sdk'; +import { NC_APP_SETTINGS } from '../../../constants'; +import Store from '../../../models/Store'; +import { Tele } from 'nc-help'; +import catchError, { NcError } from '../../helpers/catchError'; + +const { isEmail } = require('validator'); +import * as ejs from 'ejs'; + +import bcrypt from 'bcryptjs'; +import { promisify } from 'util'; +import User from '../../../models/User'; + +const { v4: uuidv4 } = require('uuid'); +import Audit from '../../../models/Audit'; +import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2'; + +import passport from 'passport'; +import extractProjectIdAndAuthenticate from '../../helpers/extractProjectIdAndAuthenticate'; +import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import { MetaTable } from '../../../utils/globals'; +import Noco from '../../../Noco'; +import { getAjvValidatorMw } from '../helpers'; +import { genJwt } from './helpers'; +import { randomTokenString } from '../../helpers/stringHelpers'; + +export async function registerNewUserIfAllowed({ + firstname, + lastname, + email, + salt, + password, + email_verification_token, + }: { + firstname; + lastname; + email: string; + salt: any; + password; + email_verification_token; +}) { + let roles: string = OrgUserRoles.CREATOR; + + if (await User.isFirst()) { + roles = `${OrgUserRoles.CREATOR},${OrgUserRoles.SUPER_ADMIN}`; + // todo: update in nc_store + // roles = 'owner,creator,editor' + Tele.emit('evt', { + evt_type: 'project:invite', + count: 1, + }); + } else { + let settings: { invite_only_signup?: boolean } = {}; + try { + settings = JSON.parse((await Store.get(NC_APP_SETTINGS))?.value); + } catch {} + + if (settings?.invite_only_signup) { + NcError.badRequest('Not allowed to signup, contact super admin.'); + } else { + roles = OrgUserRoles.VIEWER; + } + } + + const token_version = randomTokenString(); + + return await User.insert({ + firstname, + lastname, + email, + salt, + password, + email_verification_token, + roles, + token_version, + }); +} + + +export async function passwordChange(param: { + body: PasswordChangeReqType + user: UserType + req:any +}): Promise { + const { currentPassword, newPassword } = param.body; + + if (!currentPassword || !newPassword) { + return NcError.badRequest('Missing new/old password'); + } + + // validate password and throw error if password is satisfying the conditions + const { valid, error } = validatePassword(newPassword); + + if (!valid) { + NcError.badRequest(`Password : ${error}`); + } + + const user = await User.getByEmail(param.user.email); + + const hashedPassword = await promisify(bcrypt.hash)( + currentPassword, + user.salt + ); + + if (hashedPassword !== user.password) { + return NcError.badRequest('Current password is wrong'); + } + + const salt = await promisify(bcrypt.genSalt)(10); + const password = await promisify(bcrypt.hash)(newPassword, salt); + + await User.update(user.id, { + salt, + password, + email: user.email, + token_version: null, + }); + + await Audit.insert({ + op_type: 'AUTHENTICATION', + op_sub_type: 'PASSWORD_CHANGE', + user: user.email, + description: `changed password `, + ip: param.req?.clientIp, + }); + + return true +} + +export async function passwordForgot(param:{ + body: PasswordForgotReqType; + siteUrl: string; + req:any +}): Promise { + const _email = param.body.email; + + if (!_email) { + NcError.badRequest('Please enter your email address.'); + } + + const email = _email.toLowerCase(); + const user = await User.getByEmail(email); + + if (user) { + const token = uuidv4(); + await User.update(user.id, { + email: user.email, + reset_password_token: token, + reset_password_expires: new Date(Date.now() + 60 * 60 * 1000), + token_version: null, + }); + try { + const template = (await import('./ui/emailTemplates/forgotPassword')) + .default; + await NcPluginMgrv2.emailAdapter().then((adapter) => + adapter.mailSend({ + to: user.email, + subject: 'Password Reset Link', + text: `Visit following link to update your password : ${ + param.siteUrl + }/auth/password/reset/${token}.`, + html: ejs.render(template, { + resetLink: param.siteUrl + `/auth/password/reset/${token}`, + }), + }) + ); + } catch (e) { + console.log(e); + return NcError.badRequest( + 'Email Plugin is not found. Please contact administrators to configure it in App Store first.' + ); + } + + await Audit.insert({ + op_type: 'AUTHENTICATION', + op_sub_type: 'PASSWORD_FORGOT', + user: user.email, + description: `requested for password reset `, + ip: param.req?.clientIp, + }); + } else { + return NcError.badRequest('Your email has not been registered.'); + } + + return true +} + +export async function tokenValidate(param:{ + token: string; +}): Promise { + const token = param.token; + + const user = await Noco.ncMeta.metaGet(null, null, MetaTable.USERS, { + reset_password_token: token, + }); + + if (!user || !user.email) { + NcError.badRequest('Invalid reset url'); + } + if (new Date(user.reset_password_expires) < new Date()) { + NcError.badRequest('Password reset url expired'); + } + + return true +} + +export async function passwordReset(param:{ + body: PasswordResetReqType; + token: string; + // todo: exclude + req:any; +}): Promise { + const { token, body, req } = param; + + const user = await Noco.ncMeta.metaGet(null, null, MetaTable.USERS, { + reset_password_token: token, + }); + + if (!user) { + NcError.badRequest('Invalid reset url'); + } + if (user.reset_password_expires < new Date()) { + NcError.badRequest('Password reset url expired'); + } + if (user.provider && user.provider !== 'local') { + NcError.badRequest('Email registered via social account'); + } + + // validate password and throw error if password is satisfying the conditions + const { valid, error } = validatePassword(body.password); + if (!valid) { + NcError.badRequest(`Password : ${error}`); + } + + const salt = await promisify(bcrypt.genSalt)(10); + const password = await promisify(bcrypt.hash)(body.password, salt); + + await User.update(user.id, { + salt, + password, + email: user.email, + reset_password_expires: null, + reset_password_token: '', + token_version: null, + }); + + await Audit.insert({ + op_type: 'AUTHENTICATION', + op_sub_type: 'PASSWORD_RESET', + user: user.email, + description: `did reset password `, + ip: req.clientIp, + }); + + return true +} + +export async function emailVerification(param: { + token: string; + // todo: exclude + req: any; +}): Promise { + const { token, req } = param; + + const user = await Noco.ncMeta.metaGet(null, null, MetaTable.USERS, { + email_verification_token: token, + }); + + if (!user) { + NcError.badRequest('Invalid verification url'); + } + + await User.update(user.id, { + email: user.email, + email_verification_token: '', + email_verified: true, + }); + + await Audit.insert({ + op_type: 'AUTHENTICATION', + op_sub_type: 'EMAIL_VERIFICATION', + user: user.email, + description: `verified email `, + ip: req.clientIp, + }); + + return true +} + +export * from './helpers' +export * from './initAdminFromEnv' + diff --git a/packages/nocodb/src/lib/services/userService/initAdminFromEnv.ts b/packages/nocodb/src/lib/services/userService/initAdminFromEnv.ts new file mode 100644 index 0000000000..8f9bfdce74 --- /dev/null +++ b/packages/nocodb/src/lib/services/userService/initAdminFromEnv.ts @@ -0,0 +1,283 @@ +import User from '../../../models/User'; +import { v4 as uuidv4 } from 'uuid'; +import { promisify } from 'util'; + +import bcrypt from 'bcryptjs'; +import Noco from '../../../Noco'; +import { CacheScope, MetaTable } from '../../../utils/globals'; +import ProjectUser from '../../../models/ProjectUser'; +import { validatePassword } from 'nocodb-sdk'; +import boxen from 'boxen'; +import NocoCache from '../../../cache/NocoCache'; +import { Tele } from 'nc-help'; + +const { isEmail } = require('validator'); +const rolesLevel = { owner: 0, creator: 1, editor: 2, commenter: 3, viewer: 4 }; + +export default async function initAdminFromEnv(_ncMeta = Noco.ncMeta) { + if (process.env.NC_ADMIN_EMAIL && process.env.NC_ADMIN_PASSWORD) { + if (!isEmail(process.env.NC_ADMIN_EMAIL?.trim())) { + console.log( + '\n', + boxen( + `Provided admin email '${process.env.NC_ADMIN_EMAIL}' is not valid`, + { + title: 'Invalid admin email', + padding: 1, + borderStyle: 'double', + titleAlignment: 'center', + borderColor: 'red', + } + ), + '\n' + ); + process.exit(1); + } + + const { valid, error, hint } = validatePassword( + process.env.NC_ADMIN_PASSWORD + ); + if (!valid) { + console.log( + '\n', + boxen(`${error}${hint ? `\n\n${hint}` : ''}`, { + title: 'Invalid admin password', + padding: 1, + borderStyle: 'double', + titleAlignment: 'center', + borderColor: 'red', + }), + '\n' + ); + process.exit(1); + } + + let ncMeta; + try { + ncMeta = await _ncMeta.startTransaction(); + const email = process.env.NC_ADMIN_EMAIL.toLowerCase().trim(); + + const salt = await promisify(bcrypt.genSalt)(10); + const password = await promisify(bcrypt.hash)( + process.env.NC_ADMIN_PASSWORD, + salt + ); + const email_verification_token = uuidv4(); + const roles = 'user,super'; + + // if super admin not present + if (await User.isFirst(ncMeta)) { + // roles = 'owner,creator,editor' + Tele.emit('evt', { + evt_type: 'project:invite', + count: 1, + }); + + await User.insert( + { + firstname: '', + lastname: '', + email, + salt, + password, + email_verification_token, + roles, + }, + ncMeta + ); + } else { + const salt = await promisify(bcrypt.genSalt)(10); + const password = await promisify(bcrypt.hash)( + process.env.NC_ADMIN_PASSWORD, + salt + ); + const email_verification_token = uuidv4(); + const superUser = await ncMeta.metaGet2(null, null, MetaTable.USERS, { + roles: 'user,super', + }); + + if (!superUser?.id) { + const existingUserWithNewEmail = await User.getByEmail(email, ncMeta); + if (existingUserWithNewEmail?.id) { + // clear cache + await NocoCache.delAll( + CacheScope.USER, + `${existingUserWithNewEmail.email}___*` + ); + await NocoCache.del( + `${CacheScope.USER}:${existingUserWithNewEmail.id}` + ); + await NocoCache.del( + `${CacheScope.USER}:${existingUserWithNewEmail.email}` + ); + + // Update email and password of super admin account + await User.update( + existingUserWithNewEmail.id, + { + salt, + email, + password, + email_verification_token, + token_version: null, + refresh_token: null, + roles, + }, + ncMeta + ); + } else { + Tele.emit('evt', { + evt_type: 'project:invite', + count: 1, + }); + + await User.insert( + { + firstname: '', + lastname: '', + email, + salt, + password, + email_verification_token, + roles, + }, + ncMeta + ); + } + } else if (email !== superUser.email) { + // update admin email and password and migrate projects + // if user already present and associated with some project + + // check user account already present with the new admin email + const existingUserWithNewEmail = await User.getByEmail(email, ncMeta); + + if (existingUserWithNewEmail?.id) { + // get all project access belongs to the existing account + // and migrate to the admin account + const existingUserProjects = await ncMeta.metaList2( + null, + null, + MetaTable.PROJECT_USERS, + { + condition: { fk_user_id: existingUserWithNewEmail.id }, + } + ); + + for (const existingUserProject of existingUserProjects) { + const userProject = await ProjectUser.get( + existingUserProject.project_id, + superUser.id, + ncMeta + ); + + // if admin user already have access to the project + // then update role based on the highest access level + if (userProject) { + if ( + rolesLevel[userProject.roles] > + rolesLevel[existingUserProject.roles] + ) { + await ProjectUser.update( + userProject.project_id, + superUser.id, + existingUserProject.roles, + ncMeta + ); + } + } else { + // if super doesn't have access then add the access + await ProjectUser.insert( + { + ...existingUserProject, + fk_user_id: superUser.id, + }, + ncMeta + ); + } + // delete the old project access entry from DB + await ProjectUser.delete( + existingUserProject.project_id, + existingUserProject.fk_user_id, + ncMeta + ); + } + + // delete existing user + await ncMeta.metaDelete( + null, + null, + MetaTable.USERS, + existingUserWithNewEmail.id + ); + + // clear cache + await NocoCache.delAll( + CacheScope.USER, + `${existingUserWithNewEmail.email}___*` + ); + await NocoCache.del( + `${CacheScope.USER}:${existingUserWithNewEmail.id}` + ); + await NocoCache.del( + `${CacheScope.USER}:${existingUserWithNewEmail.email}` + ); + + // Update email and password of super admin account + await User.update( + superUser.id, + { + salt, + email, + password, + email_verification_token, + token_version: null, + refresh_token: null, + }, + ncMeta + ); + } else { + // if email's are not different update the password and hash + await User.update( + superUser.id, + { + salt, + email, + password, + email_verification_token, + token_version: null, + refresh_token: null, + }, + ncMeta + ); + } + } else { + const newPasswordHash = await promisify(bcrypt.hash)( + process.env.NC_ADMIN_PASSWORD, + superUser.salt + ); + + if (newPasswordHash !== superUser.password) { + // if email's are same and passwords are different + // then update the password and token version + await User.update( + superUser.id, + { + salt, + password, + email_verification_token, + token_version: null, + refresh_token: null, + }, + ncMeta + ); + } + } + } + await ncMeta.commit(); + } catch (e) { + console.log('Error occurred while updating/creating admin user'); + console.log(e); + await ncMeta.rollback(e); + } + } +} From ed939174fe3414ab0eed604d572ce15feb8d186c Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 11:38:59 +0530 Subject: [PATCH 20/64] refactor: sync api Signed-off-by: Pranav C --- .../controllers/syncController/importApis.ts | 142 + .../lib/controllers/syncController/index.ts | 69 + .../lib/controllers/userController/index.ts | 462 +++- .../controllers/userController/userApis.ts | 462 ---- packages/nocodb/src/lib/services/index.ts | 1 + .../services/syncService/helpers/EntityMap.ts | 222 ++ .../helpers/NocoSyncDestAdapter.ts | 6 + .../helpers/NocoSyncSourceAdapter.ts | 7 + .../services/syncService/helpers/fetchAT.ts | 238 ++ .../lib/services/syncService/helpers/job.ts | 2430 +++++++++++++++++ .../syncService/helpers/readAndProcessData.ts | 338 +++ .../services/syncService/helpers/syncMap.ts | 31 + .../src/lib/services/syncService/index.ts | 47 + 13 files changed, 3992 insertions(+), 463 deletions(-) create mode 100644 packages/nocodb/src/lib/controllers/syncController/importApis.ts create mode 100644 packages/nocodb/src/lib/controllers/syncController/index.ts delete mode 100644 packages/nocodb/src/lib/controllers/userController/userApis.ts create mode 100644 packages/nocodb/src/lib/services/syncService/helpers/EntityMap.ts create mode 100644 packages/nocodb/src/lib/services/syncService/helpers/NocoSyncDestAdapter.ts create mode 100644 packages/nocodb/src/lib/services/syncService/helpers/NocoSyncSourceAdapter.ts create mode 100644 packages/nocodb/src/lib/services/syncService/helpers/fetchAT.ts create mode 100644 packages/nocodb/src/lib/services/syncService/helpers/job.ts create mode 100644 packages/nocodb/src/lib/services/syncService/helpers/readAndProcessData.ts create mode 100644 packages/nocodb/src/lib/services/syncService/helpers/syncMap.ts create mode 100644 packages/nocodb/src/lib/services/syncService/index.ts diff --git a/packages/nocodb/src/lib/controllers/syncController/importApis.ts b/packages/nocodb/src/lib/controllers/syncController/importApis.ts new file mode 100644 index 0000000000..f161a40ada --- /dev/null +++ b/packages/nocodb/src/lib/controllers/syncController/importApis.ts @@ -0,0 +1,142 @@ +import { Request, Router } from 'express'; +// import { Queue } from 'bullmq'; +// import axios from 'axios'; +import catchError, { NcError } from '../../meta/helpers/catchError'; +import { Server } from 'socket.io'; +import NocoJobs from '../../jobs/NocoJobs'; +import { SyncSource } from '../../models'; +import Noco from '../../Noco'; +import { syncService, userService } from '../../services'; +import { AirtableSyncConfig } from '../../services/syncService/helpers/job'; + +const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB'; +const AIRTABLE_PROGRESS_JOB = 'AIRTABLE_PROGRESS_JOB'; + +enum SyncStatus { + PROGRESS = 'PROGRESS', + COMPLETED = 'COMPLETED', + FAILED = 'FAILED', +} + +export default ( + router: Router, + sv: Server, + jobs: { [id: string]: { last_message: any } } +) => { + // add importer job handler and progress notification job handler + NocoJobs.jobsMgr.addJobWorker( + AIRTABLE_IMPORT_JOB, + syncService.airtableImportJob + ); + NocoJobs.jobsMgr.addJobWorker( + AIRTABLE_PROGRESS_JOB, + ({ payload, progress }) => { + sv.to(payload?.id).emit('progress', { + msg: progress?.msg, + level: progress?.level, + status: progress?.status, + }); + + if (payload?.id in jobs) { + jobs[payload?.id].last_message = { + msg: progress?.msg, + level: progress?.level, + status: progress?.status, + }; + } + } + ); + + NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => { + NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { + payload, + progress: { + msg: progress?.msg, + level: progress?.level, + status: progress?.status, + }, + }); + }); + NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, (payload) => { + NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { + payload, + progress: { + msg: 'Complete!', + status: SyncStatus.COMPLETED, + }, + }); + delete jobs[payload?.id]; + }); + NocoJobs.jobsMgr.addFailureCbk(AIRTABLE_IMPORT_JOB, (payload, error: any) => { + NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { + payload, + progress: { + msg: error?.message || 'Failed due to some internal error', + status: SyncStatus.FAILED, + }, + }); + delete jobs[payload?.id]; + }); + + router.post( + '/api/v1/db/meta/import/airtable', + catchError((req, res) => { + NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { + id: req.query.id, + ...req.body, + }); + res.json({}); + }) + ); + router.post( + '/api/v1/db/meta/syncs/:syncId/trigger', + catchError(async (req: Request, res) => { + if (req.params.syncId in jobs) { + NcError.badRequest('Sync already in progress'); + } + + const syncSource = await SyncSource.get(req.params.syncId); + + const user = await syncSource.getUser(); + const token = userService.genJwt(user, Noco.getConfig()); + + // Treat default baseUrl as siteUrl from req object + let baseURL = (req as any).ncSiteUrl; + + // if environment value avail use it + // or if it's docker construct using `PORT` + if (process.env.NC_BASEURL_INTERNAL) { + baseURL = process.env.NC_BASEURL_INTERNAL; + } else if (process.env.NC_DOCKER) { + baseURL = `http://localhost:${process.env.PORT || 8080}`; + } + + setTimeout(() => { + NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { + id: req.params.syncId, + ...(syncSource?.details || {}), + projectId: syncSource.project_id, + baseId: syncSource.base_id, + authToken: token, + baseURL, + }); + }, 1000); + + jobs[req.params.syncId] = { + last_message: { + msg: 'Sync started', + }, + }; + res.json({}); + }) + ); + router.post( + '/api/v1/db/meta/syncs/:syncId/abort', + catchError(async (req: Request, res) => { + if (req.params.syncId in jobs) { + delete jobs[req.params.syncId]; + } + res.json({}); + }) + ); +}; diff --git a/packages/nocodb/src/lib/controllers/syncController/index.ts b/packages/nocodb/src/lib/controllers/syncController/index.ts new file mode 100644 index 0000000000..7dbfac49b5 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/syncController/index.ts @@ -0,0 +1,69 @@ +import { Request, Response, Router } from 'express'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import { syncService } from '../../services'; + +export async function syncSourceList(req: Request, res: Response) { + // todo: pagination + res.json( + syncService.syncSourceList({ + projectId: req.params.projectId, + }) + ); +} + +export async function syncCreate(req: Request, res: Response) { + res.json( + await syncService.syncCreate({ + projectId: req.params.projectId, + baseId: req.params.baseId, + userId: (req as any).user.id, + syncPayload: req.body, + }) + ); +} + +export async function syncDelete(req: Request, res: Response) { + res.json( + await syncService.syncDelete({ + syncId: req.params.syncId, + }) + ); +} + +export async function syncUpdate(req: Request, res: Response) { + res.json( + await syncService.syncUpdate({ + syncId: req.params.syncId, + syncPayload: req.body, + }) + ); +} + +const router = Router({ mergeParams: true }); + +router.get( + '/api/v1/db/meta/projects/:projectId/syncs', + ncMetaAclMw(syncSourceList, 'syncSourceList') +); +router.post( + '/api/v1/db/meta/projects/:projectId/syncs', + ncMetaAclMw(syncCreate, 'syncSourceCreate') +); +router.get( + '/api/v1/db/meta/projects/:projectId/syncs/:baseId', + ncMetaAclMw(syncSourceList, 'syncSourceList') +); +router.post( + '/api/v1/db/meta/projects/:projectId/syncs/:baseId', + ncMetaAclMw(syncCreate, 'syncSourceCreate') +); +router.delete( + '/api/v1/db/meta/syncs/:syncId', + ncMetaAclMw(syncDelete, 'syncSourceDelete') +); +router.patch( + '/api/v1/db/meta/syncs/:syncId', + ncMetaAclMw(syncUpdate, 'syncSourceUpdate') +); + +export default router; diff --git a/packages/nocodb/src/lib/controllers/userController/index.ts b/packages/nocodb/src/lib/controllers/userController/index.ts index c4a5003430..29a53d781e 100644 --- a/packages/nocodb/src/lib/controllers/userController/index.ts +++ b/packages/nocodb/src/lib/controllers/userController/index.ts @@ -1 +1,461 @@ -export * from './userApis'; +import { Request, Response } from 'express'; +import { TableType, validatePassword } from 'nocodb-sdk'; +import { Tele } from 'nc-help'; + +const { isEmail } = require('validator'); +import * as ejs from 'ejs'; + +import bcrypt from 'bcryptjs'; +import { promisify } from 'util'; + +const { v4: uuidv4 } = require('uuid'); + +import passport from 'passport'; +import { getAjvValidatorMw } from '../../meta/api/helpers'; +import catchError, { NcError } from '../../meta/helpers/catchError'; +import extractProjectIdAndAuthenticate from '../../meta/helpers/extractProjectIdAndAuthenticate'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; +import { Audit, User } from '../../models'; +import Noco from '../../Noco'; +import { userService } from '../../services'; + +export async function signup(req: Request, res: Response) { + const { + email: _email, + firstname, + lastname, + token, + ignore_subscribe, + } = req.body; + let { password } = req.body; + + // validate password and throw error if password is satisfying the conditions + const { valid, error } = validatePassword(password); + if (!valid) { + NcError.badRequest(`Password : ${error}`); + } + + if (!isEmail(_email)) { + NcError.badRequest(`Invalid email`); + } + + const email = _email.toLowerCase(); + + let user = await User.getByEmail(email); + + if (user) { + if (token) { + if (token !== user.invite_token) { + NcError.badRequest(`Invalid invite url`); + } else if (user.invite_token_expires < new Date()) { + NcError.badRequest( + 'Expired invite url, Please contact super admin to get a new invite url' + ); + } + } else { + // todo : opening up signup for timebeing + // return next(new Error(`Email '${email}' already registered`)); + } + } + + const salt = await promisify(bcrypt.genSalt)(10); + password = await promisify(bcrypt.hash)(password, salt); + const email_verification_token = uuidv4(); + + if (!ignore_subscribe) { + Tele.emit('evt_subscribe', email); + } + + if (user) { + if (token) { + await User.update(user.id, { + firstname, + lastname, + salt, + password, + email_verification_token, + invite_token: null, + invite_token_expires: null, + email: user.email, + }); + } else { + NcError.badRequest('User already exist'); + } + } else { + await userService.registerNewUserIfAllowed({ + firstname, + lastname, + email, + salt, + password, + email_verification_token, + }); + } + user = await User.getByEmail(email); + + try { + const template = (await import('./ui/emailTemplates/verify')).default; + await ( + await NcPluginMgrv2.emailAdapter() + ).mailSend({ + to: email, + subject: 'Verify email', + html: ejs.render(template, { + verifyLink: + (req as any).ncSiteUrl + + `/email/verify/${user.email_verification_token}`, + }), + }); + } catch (e) { + console.log( + 'Warning : `mailSend` failed, Please configure emailClient configuration.' + ); + } + await promisify((req as any).login.bind(req))(user); + const refreshToken = userService.randomTokenString(); + await User.update(user.id, { + refresh_token: refreshToken, + email: user.email, + }); + + setTokenCookie(res, refreshToken); + + user = (req as any).user; + + await Audit.insert({ + op_type: 'AUTHENTICATION', + op_sub_type: 'SIGNUP', + user: user.email, + description: `signed up `, + ip: (req as any).clientIp, + }); + + res.json({ + token: userService.genJwt(user, Noco.getConfig()), + } as any); +} + +async function successfulSignIn({ + user, + err, + info, + req, + res, + auditDescription, +}) { + try { + if (!user || !user.email) { + if (err) { + return res.status(400).send(err); + } + if (info) { + return res.status(400).send(info); + } + return res.status(400).send({ msg: 'Your signin has failed' }); + } + + await promisify((req as any).login.bind(req))(user); + const refreshToken = userService.randomTokenString(); + + if (!user.token_version) { + user.token_version = userService.randomTokenString(); + } + + await User.update(user.id, { + refresh_token: refreshToken, + email: user.email, + token_version: user.token_version, + }); + setTokenCookie(res, refreshToken); + + await Audit.insert({ + op_type: 'AUTHENTICATION', + op_sub_type: 'SIGNIN', + user: user.email, + ip: req.clientIp, + description: auditDescription, + }); + + res.json({ + token: userService.genJwt(user, Noco.getConfig()), + } as any); + } catch (e) { + console.log(e); + throw e; + } +} + +async function signin(req, res, next) { + passport.authenticate( + 'local', + { session: false }, + async (err, user, info): Promise => + await successfulSignIn({ + user, + err, + info, + req, + res, + auditDescription: 'signed in', + }) + )(req, res, next); +} + +async function googleSignin(req, res, next) { + passport.authenticate( + 'google', + { + session: false, + callbackURL: req.ncSiteUrl + Noco.getConfig().dashboardPath, + }, + async (err, user, info): Promise => + await successfulSignIn({ + user, + err, + info, + req, + res, + auditDescription: 'signed in using Google Auth', + }) + )(req, res, next); +} + +function setTokenCookie(res: Response, token): void { + // create http only cookie with refresh token that expires in 7 days + const cookieOptions = { + httpOnly: true, + expires: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000), + }; + res.cookie('refresh_token', token, cookieOptions); +} + +async function me(req, res): Promise { + res.json(req?.session?.passport?.user ?? {}); +} + +async function passwordChange(req: Request, res): Promise { + if (!(req as any).isAuthenticated()) { + NcError.forbidden('Not allowed'); + } + + await userService.passwordChange({ + user: req['user'], + req, + body: req.body, + }); + + res.json({ msg: 'Password updated successfully' }); +} + +async function passwordForgot(req: Request, res): Promise { + await userService.passwordForgot({ + siteUrl: (req as any).ncSiteUrl, + body: req.body, + req, + }); + + res.json({ msg: 'Please check your email to reset the password' }); +} + +async function tokenValidate(req, res): Promise { + await userService.tokenValidate({ + token: req.params.tokenId, + }); + res.json(true); +} + +async function passwordReset(req, res): Promise { + await userService.passwordReset({ + token: req.params.tokenId, + body: req.body, + req, + }); + + res.json({ msg: 'Password reset successful' }); +} + +async function emailVerification(req, res): Promise { + await userService.emailVerification({ + token: req.params.tokenId, + req, + }); + + res.json({ msg: 'Email verified successfully' }); +} + +async function refreshToken(req, res): Promise { + try { + if (!req?.cookies?.refresh_token) { + return res.status(400).json({ msg: 'Missing refresh token' }); + } + + const user = await User.getByRefreshToken(req.cookies.refresh_token); + + if (!user) { + return res.status(400).json({ msg: 'Invalid refresh token' }); + } + + const refreshToken = userService.randomTokenString(); + + await User.update(user.id, { + email: user.email, + refresh_token: refreshToken, + }); + + setTokenCookie(res, refreshToken); + + res.json({ + token: userService.genJwt(user, Noco.getConfig()), + } as any); + } catch (e) { + return res.status(400).json({ msg: e.message }); + } +} + +async function renderPasswordReset(req, res): Promise { + try { + res.send( + ejs.render((await import('./ui/auth/resetPassword')).default, { + ncPublicUrl: process.env.NC_PUBLIC_URL || '', + token: JSON.stringify(req.params.tokenId), + baseUrl: `/`, + }) + ); + } catch (e) { + return res.status(400).json({ msg: e.message }); + } +} + +const mapRoutes = (router) => { + // todo: old api - /auth/signup?tool=1 + router.post( + '/auth/user/signup', + getAjvValidatorMw('swagger.json#/components/schemas/SignUpReq'), + catchError(signup) + ); + router.post( + '/auth/user/signin', + getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), + catchError(signin) + ); + router.get('/auth/user/me', extractProjectIdAndAuthenticate, catchError(me)); + router.post( + '/auth/password/forgot', + getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), + catchError(passwordForgot) + ); + router.post('/auth/token/validate/:tokenId', catchError(tokenValidate)); + router.post( + '/auth/password/reset/:tokenId', + getAjvValidatorMw('swagger.json#/components/schemas/PasswordResetReq'), + catchError(passwordReset) + ); + router.post('/auth/email/validate/:tokenId', catchError(emailVerification)); + router.post( + '/user/password/change', + getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), + ncMetaAclMw(passwordChange, 'passwordChange') + ); + router.post('/auth/token/refresh', catchError(refreshToken)); + + /* Google auth apis */ + + router.post(`/auth/google/genTokenByCode`, catchError(googleSignin)); + + router.get('/auth/google', (req: any, res, next) => + passport.authenticate('google', { + scope: ['profile', 'email'], + state: req.query.state, + callbackURL: req.ncSiteUrl + Noco.getConfig().dashboardPath, + })(req, res, next) + ); + + // deprecated APIs + router.post( + '/api/v1/db/auth/user/signup', + getAjvValidatorMw('swagger.json#/components/schemas/SignUpReq'), + catchError(signup) + ); + router.post( + '/api/v1/db/auth/user/signin', + getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), + catchError(signin) + ); + router.get( + '/api/v1/db/auth/user/me', + extractProjectIdAndAuthenticate, + catchError(me) + ); + router.post( + '/api/v1/db/auth/password/forgot', + getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), + catchError(passwordForgot) + ); + router.post( + '/api/v1/db/auth/token/validate/:tokenId', + catchError(tokenValidate) + ); + router.post( + '/api/v1/db/auth/password/reset/:tokenId', + getAjvValidatorMw('swagger.json#/components/schemas/PasswordResetReq'), + catchError(passwordReset) + ); + router.post( + '/api/v1/db/auth/email/validate/:tokenId', + catchError(emailVerification) + ); + router.post( + '/api/v1/db/auth/password/change', + getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), + ncMetaAclMw(passwordChange, 'passwordChange') + ); + router.post('/api/v1/db/auth/token/refresh', catchError(refreshToken)); + router.get( + '/api/v1/db/auth/password/reset/:tokenId', + catchError(renderPasswordReset) + ); + + // new API + router.post( + '/api/v1/auth/user/signup', + getAjvValidatorMw('swagger.json#/components/schemas/SignUpReq'), + catchError(signup) + ); + router.post( + '/api/v1/auth/user/signin', + getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), + catchError(signin) + ); + router.get( + '/api/v1/auth/user/me', + extractProjectIdAndAuthenticate, + catchError(me) + ); + router.post( + '/api/v1/auth/password/forgot', + getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), + catchError(passwordForgot) + ); + router.post( + '/api/v1/auth/token/validate/:tokenId', + catchError(tokenValidate) + ); + router.post( + '/api/v1/auth/password/reset/:tokenId', + catchError(passwordReset) + ); + router.post( + '/api/v1/auth/email/validate/:tokenId', + catchError(emailVerification) + ); + router.post( + '/api/v1/auth/password/change', + getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), + ncMetaAclMw(passwordChange, 'passwordChange') + ); + router.post('/api/v1/auth/token/refresh', catchError(refreshToken)); + // respond with password reset page + router.get('/auth/password/reset/:tokenId', catchError(renderPasswordReset)); +}; +export { mapRoutes as userApis }; diff --git a/packages/nocodb/src/lib/controllers/userController/userApis.ts b/packages/nocodb/src/lib/controllers/userController/userApis.ts deleted file mode 100644 index 367eb4591b..0000000000 --- a/packages/nocodb/src/lib/controllers/userController/userApis.ts +++ /dev/null @@ -1,462 +0,0 @@ -import { Request, Response } from 'express'; -import { TableType, validatePassword } from 'nocodb-sdk'; -import { Tele } from 'nc-help'; - -const { isEmail } = require('validator'); -import * as ejs from 'ejs'; - -import bcrypt from 'bcryptjs'; -import { promisify } from 'util'; - -const { v4: uuidv4 } = require('uuid'); - -import passport from 'passport'; -import { getAjvValidatorMw } from '../../meta/api/helpers'; -import catchError, { NcError } from '../../meta/helpers/catchError'; -import extractProjectIdAndAuthenticate from '../../meta/helpers/extractProjectIdAndAuthenticate'; -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; -import { Audit, User } from '../../models'; -import Noco from '../../Noco'; -import { genJwt } from './helpers'; -import { userService } from '../../services'; - -export async function signup(req: Request, res: Response) { - const { - email: _email, - firstname, - lastname, - token, - ignore_subscribe, - } = req.body; - let { password } = req.body; - - // validate password and throw error if password is satisfying the conditions - const { valid, error } = validatePassword(password); - if (!valid) { - NcError.badRequest(`Password : ${error}`); - } - - if (!isEmail(_email)) { - NcError.badRequest(`Invalid email`); - } - - const email = _email.toLowerCase(); - - let user = await User.getByEmail(email); - - if (user) { - if (token) { - if (token !== user.invite_token) { - NcError.badRequest(`Invalid invite url`); - } else if (user.invite_token_expires < new Date()) { - NcError.badRequest( - 'Expired invite url, Please contact super admin to get a new invite url' - ); - } - } else { - // todo : opening up signup for timebeing - // return next(new Error(`Email '${email}' already registered`)); - } - } - - const salt = await promisify(bcrypt.genSalt)(10); - password = await promisify(bcrypt.hash)(password, salt); - const email_verification_token = uuidv4(); - - if (!ignore_subscribe) { - Tele.emit('evt_subscribe', email); - } - - if (user) { - if (token) { - await User.update(user.id, { - firstname, - lastname, - salt, - password, - email_verification_token, - invite_token: null, - invite_token_expires: null, - email: user.email, - }); - } else { - NcError.badRequest('User already exist'); - } - } else { - await userService.registerNewUserIfAllowed({ - firstname, - lastname, - email, - salt, - password, - email_verification_token, - }); - } - user = await User.getByEmail(email); - - try { - const template = (await import('./ui/emailTemplates/verify')).default; - await ( - await NcPluginMgrv2.emailAdapter() - ).mailSend({ - to: email, - subject: 'Verify email', - html: ejs.render(template, { - verifyLink: - (req as any).ncSiteUrl + - `/email/verify/${user.email_verification_token}`, - }), - }); - } catch (e) { - console.log( - 'Warning : `mailSend` failed, Please configure emailClient configuration.' - ); - } - await promisify((req as any).login.bind(req))(user); - const refreshToken = userService.randomTokenString(); - await User.update(user.id, { - refresh_token: refreshToken, - email: user.email, - }); - - setTokenCookie(res, refreshToken); - - user = (req as any).user; - - await Audit.insert({ - op_type: 'AUTHENTICATION', - op_sub_type: 'SIGNUP', - user: user.email, - description: `signed up `, - ip: (req as any).clientIp, - }); - - res.json({ - token: genJwt(user, Noco.getConfig()), - } as any); -} - -async function successfulSignIn({ - user, - err, - info, - req, - res, - auditDescription, -}) { - try { - if (!user || !user.email) { - if (err) { - return res.status(400).send(err); - } - if (info) { - return res.status(400).send(info); - } - return res.status(400).send({ msg: 'Your signin has failed' }); - } - - await promisify((req as any).login.bind(req))(user); - const refreshToken = userService.randomTokenString(); - - if (!user.token_version) { - user.token_version = userService.randomTokenString(); - } - - await User.update(user.id, { - refresh_token: refreshToken, - email: user.email, - token_version: user.token_version, - }); - setTokenCookie(res, refreshToken); - - await Audit.insert({ - op_type: 'AUTHENTICATION', - op_sub_type: 'SIGNIN', - user: user.email, - ip: req.clientIp, - description: auditDescription, - }); - - res.json({ - token: genJwt(user, Noco.getConfig()), - } as any); - } catch (e) { - console.log(e); - throw e; - } -} - -async function signin(req, res, next) { - passport.authenticate( - 'local', - { session: false }, - async (err, user, info): Promise => - await successfulSignIn({ - user, - err, - info, - req, - res, - auditDescription: 'signed in', - }) - )(req, res, next); -} - -async function googleSignin(req, res, next) { - passport.authenticate( - 'google', - { - session: false, - callbackURL: req.ncSiteUrl + Noco.getConfig().dashboardPath, - }, - async (err, user, info): Promise => - await successfulSignIn({ - user, - err, - info, - req, - res, - auditDescription: 'signed in using Google Auth', - }) - )(req, res, next); -} - -function setTokenCookie(res: Response, token): void { - // create http only cookie with refresh token that expires in 7 days - const cookieOptions = { - httpOnly: true, - expires: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000), - }; - res.cookie('refresh_token', token, cookieOptions); -} - -async function me(req, res): Promise { - res.json(req?.session?.passport?.user ?? {}); -} - -async function passwordChange(req: Request, res): Promise { - if (!(req as any).isAuthenticated()) { - NcError.forbidden('Not allowed'); - } - - await userService.passwordChange({ - user: req['user'], - req, - body: req.body, - }); - - res.json({ msg: 'Password updated successfully' }); -} - -async function passwordForgot(req: Request, res): Promise { - await userService.passwordForgot({ - siteUrl: (req as any).ncSiteUrl, - body: req.body, - req, - }); - - res.json({ msg: 'Please check your email to reset the password' }); -} - -async function tokenValidate(req, res): Promise { - await userService.tokenValidate({ - token: req.params.tokenId, - }); - res.json(true); -} - -async function passwordReset(req, res): Promise { - await userService.passwordReset({ - token: req.params.tokenId, - body: req.body, - req, - }); - - res.json({ msg: 'Password reset successful' }); -} - -async function emailVerification(req, res): Promise { - await userService.emailVerification({ - token: req.params.tokenId, - req, - }); - - res.json({ msg: 'Email verified successfully' }); -} - -async function refreshToken(req, res): Promise { - try { - if (!req?.cookies?.refresh_token) { - return res.status(400).json({ msg: 'Missing refresh token' }); - } - - const user = await User.getByRefreshToken(req.cookies.refresh_token); - - if (!user) { - return res.status(400).json({ msg: 'Invalid refresh token' }); - } - - const refreshToken = userService.randomTokenString(); - - await User.update(user.id, { - email: user.email, - refresh_token: refreshToken, - }); - - setTokenCookie(res, refreshToken); - - res.json({ - token: genJwt(user, Noco.getConfig()), - } as any); - } catch (e) { - return res.status(400).json({ msg: e.message }); - } -} - -async function renderPasswordReset(req, res): Promise { - try { - res.send( - ejs.render((await import('./ui/auth/resetPassword')).default, { - ncPublicUrl: process.env.NC_PUBLIC_URL || '', - token: JSON.stringify(req.params.tokenId), - baseUrl: `/`, - }) - ); - } catch (e) { - return res.status(400).json({ msg: e.message }); - } -} - -const mapRoutes = (router) => { - // todo: old api - /auth/signup?tool=1 - router.post( - '/auth/user/signup', - getAjvValidatorMw('swagger.json#/components/schemas/SignUpReq'), - catchError(signup) - ); - router.post( - '/auth/user/signin', - getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), - catchError(signin) - ); - router.get('/auth/user/me', extractProjectIdAndAuthenticate, catchError(me)); - router.post( - '/auth/password/forgot', - getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), - catchError(passwordForgot) - ); - router.post('/auth/token/validate/:tokenId', catchError(tokenValidate)); - router.post( - '/auth/password/reset/:tokenId', - getAjvValidatorMw('swagger.json#/components/schemas/PasswordResetReq'), - catchError(passwordReset) - ); - router.post('/auth/email/validate/:tokenId', catchError(emailVerification)); - router.post( - '/user/password/change', - getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), - ncMetaAclMw(passwordChange, 'passwordChange') - ); - router.post('/auth/token/refresh', catchError(refreshToken)); - - /* Google auth apis */ - - router.post(`/auth/google/genTokenByCode`, catchError(googleSignin)); - - router.get('/auth/google', (req: any, res, next) => - passport.authenticate('google', { - scope: ['profile', 'email'], - state: req.query.state, - callbackURL: req.ncSiteUrl + Noco.getConfig().dashboardPath, - })(req, res, next) - ); - - // deprecated APIs - router.post( - '/api/v1/db/auth/user/signup', - getAjvValidatorMw('swagger.json#/components/schemas/SignUpReq'), - catchError(signup) - ); - router.post( - '/api/v1/db/auth/user/signin', - getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), - catchError(signin) - ); - router.get( - '/api/v1/db/auth/user/me', - extractProjectIdAndAuthenticate, - catchError(me) - ); - router.post( - '/api/v1/db/auth/password/forgot', - getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), - catchError(passwordForgot) - ); - router.post( - '/api/v1/db/auth/token/validate/:tokenId', - catchError(tokenValidate) - ); - router.post( - '/api/v1/db/auth/password/reset/:tokenId', - getAjvValidatorMw('swagger.json#/components/schemas/PasswordResetReq'), - catchError(passwordReset) - ); - router.post( - '/api/v1/db/auth/email/validate/:tokenId', - catchError(emailVerification) - ); - router.post( - '/api/v1/db/auth/password/change', - getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), - ncMetaAclMw(passwordChange, 'passwordChange') - ); - router.post('/api/v1/db/auth/token/refresh', catchError(refreshToken)); - router.get( - '/api/v1/db/auth/password/reset/:tokenId', - catchError(renderPasswordReset) - ); - - // new API - router.post( - '/api/v1/auth/user/signup', - getAjvValidatorMw('swagger.json#/components/schemas/SignUpReq'), - catchError(signup) - ); - router.post( - '/api/v1/auth/user/signin', - getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), - catchError(signin) - ); - router.get( - '/api/v1/auth/user/me', - extractProjectIdAndAuthenticate, - catchError(me) - ); - router.post( - '/api/v1/auth/password/forgot', - getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), - catchError(passwordForgot) - ); - router.post( - '/api/v1/auth/token/validate/:tokenId', - catchError(tokenValidate) - ); - router.post( - '/api/v1/auth/password/reset/:tokenId', - catchError(passwordReset) - ); - router.post( - '/api/v1/auth/email/validate/:tokenId', - catchError(emailVerification) - ); - router.post( - '/api/v1/auth/password/change', - getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), - ncMetaAclMw(passwordChange, 'passwordChange') - ); - router.post('/api/v1/auth/token/refresh', catchError(refreshToken)); - // respond with password reset page - router.get('/auth/password/reset/:tokenId', catchError(renderPasswordReset)); -}; -export { mapRoutes as userApis }; diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index d0fe29bdcf..159a24f942 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -32,3 +32,4 @@ export * as cacheService from './cacheService'; export * as auditService from './auditService'; export * as swaggerService from './swaggerService'; export * as userService from './userService'; +export * as syncService from './syncService'; diff --git a/packages/nocodb/src/lib/services/syncService/helpers/EntityMap.ts b/packages/nocodb/src/lib/services/syncService/helpers/EntityMap.ts new file mode 100644 index 0000000000..bd0bdd86ca --- /dev/null +++ b/packages/nocodb/src/lib/services/syncService/helpers/EntityMap.ts @@ -0,0 +1,222 @@ +import sqlite3 from 'sqlite3'; +import { Readable } from 'stream'; + +class EntityMap { + initialized: boolean; + cols: string[]; + db: any; + + constructor(...args) { + this.initialized = false; + this.cols = args.map((arg) => processKey(arg)); + this.db = new Promise((resolve, reject) => { + const db = new sqlite3.Database(':memory:'); + + const colStatement = + this.cols.length > 0 + ? this.cols.join(' TEXT, ') + ' TEXT' + : 'mappingPlaceholder TEXT'; + db.run(`CREATE TABLE mapping (${colStatement})`, (err) => { + if (err) { + console.log(err); + reject(err); + } + resolve(db); + }); + }); + } + + async init() { + if (!this.initialized) { + this.db = await this.db; + this.initialized = true; + } + } + + destroy() { + if (this.initialized && this.db) { + this.db.close(); + } + } + + async addRow(row) { + if (!this.initialized) { + throw 'Please initialize first!'; + } + + const cols = Object.keys(row).map((key) => processKey(key)); + const colStatement = cols.map((key) => `'${key}'`).join(', '); + const questionMarks = cols.map(() => '?').join(', '); + + const promises = []; + + for (const col of cols.filter((col) => !this.cols.includes(col))) { + promises.push( + new Promise((resolve, reject) => { + this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => { + if (err) { + console.log(err); + reject(err); + } + this.cols.push(col); + resolve(true); + }); + }) + ); + } + + await Promise.all(promises); + + const values = Object.values(row).map((val) => { + if (typeof val === 'object') { + return `JSON::${JSON.stringify(val)}`; + } + return val; + }); + + return new Promise((resolve, reject) => { + this.db.run( + `INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`, + values, + (err) => { + if (err) { + console.log(err); + reject(err); + } + resolve(true); + } + ); + }); + } + + getRow(col, val, res = []): Promise> { + if (!this.initialized) { + throw 'Please initialize first!'; + } + return new Promise((resolve, reject) => { + col = processKey(col); + res = res.map((r) => processKey(r)); + this.db.get( + `SELECT ${ + res.length ? res.join(', ') : '*' + } FROM mapping WHERE ${col} = ?`, + [val], + (err, rs) => { + if (err) { + console.log(err); + reject(err); + } + if (rs) { + rs = processResponseRow(rs); + } + resolve(rs); + } + ); + }); + } + + getCount(): Promise { + if (!this.initialized) { + throw 'Please initialize first!'; + } + return new Promise((resolve, reject) => { + this.db.get(`SELECT COUNT(*) as count FROM mapping`, (err, rs) => { + if (err) { + console.log(err); + reject(err); + } + resolve(rs.count); + }); + }); + } + + getStream(res = []): DBStream { + if (!this.initialized) { + throw 'Please initialize first!'; + } + res = res.map((r) => processKey(r)); + return new DBStream( + this.db, + `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping` + ); + } + + getLimit(limit, offset, res = []): Promise[]> { + if (!this.initialized) { + throw 'Please initialize first!'; + } + return new Promise((resolve, reject) => { + res = res.map((r) => processKey(r)); + this.db.all( + `SELECT ${ + res.length ? res.join(', ') : '*' + } FROM mapping LIMIT ${limit} OFFSET ${offset}`, + (err, rs) => { + if (err) { + console.log(err); + reject(err); + } + for (let row of rs) { + row = processResponseRow(row); + } + resolve(rs); + } + ); + }); + } +} + +class DBStream extends Readable { + db: any; + stmt: any; + sql: any; + + constructor(db, sql) { + super({ objectMode: true }); + this.db = db; + this.sql = sql; + this.stmt = this.db.prepare(this.sql); + this.on('end', () => this.stmt.finalize()); + } + + _read() { + let stream = this; + this.stmt.get(function (err, result) { + if (err) { + stream.emit('error', err); + } else { + if (result) { + result = processResponseRow(result); + } + stream.push(result || null); + } + }); + } +} + +function processResponseRow(res: any) { + for (const key of Object.keys(res)) { + if (res[key] && res[key].startsWith('JSON::')) { + try { + res[key] = JSON.parse(res[key].replace('JSON::', '')); + } catch (e) { + console.log(e); + } + } + if (revertKey(key) !== key) { + res[revertKey(key)] = res[key]; + delete res[key]; + } + } + return res; +} + +function processKey(key) { + return key.replace(/'/g, "''").replace(/[A-Z]/g, (match) => `_${match}`); +} + +function revertKey(key) { + return key.replace(/''/g, "'").replace(/_[A-Z]/g, (match) => match[1]); +} + +export default EntityMap; diff --git a/packages/nocodb/src/lib/services/syncService/helpers/NocoSyncDestAdapter.ts b/packages/nocodb/src/lib/services/syncService/helpers/NocoSyncDestAdapter.ts new file mode 100644 index 0000000000..8af4a493c5 --- /dev/null +++ b/packages/nocodb/src/lib/services/syncService/helpers/NocoSyncDestAdapter.ts @@ -0,0 +1,6 @@ +export abstract class NocoSyncSourceAdapter { + public abstract init(): Promise; + public abstract destProjectWrite(): Promise; + public abstract destSchemaWrite(): Promise; + public abstract destDataWrite(): Promise; +} diff --git a/packages/nocodb/src/lib/services/syncService/helpers/NocoSyncSourceAdapter.ts b/packages/nocodb/src/lib/services/syncService/helpers/NocoSyncSourceAdapter.ts new file mode 100644 index 0000000000..a419d1c24d --- /dev/null +++ b/packages/nocodb/src/lib/services/syncService/helpers/NocoSyncSourceAdapter.ts @@ -0,0 +1,7 @@ +export abstract class NocoSyncSourceAdapter { + public abstract init(): Promise; + public abstract srcSchemaGet(): Promise; + public abstract srcDataLoad(): Promise; + public abstract srcDataListen(): Promise; + public abstract srcDataPoll(): Promise; +} diff --git a/packages/nocodb/src/lib/services/syncService/helpers/fetchAT.ts b/packages/nocodb/src/lib/services/syncService/helpers/fetchAT.ts new file mode 100644 index 0000000000..4ada40bd17 --- /dev/null +++ b/packages/nocodb/src/lib/services/syncService/helpers/fetchAT.ts @@ -0,0 +1,238 @@ +const axios = require('axios').default; + +const info: any = { + initialized: false, +}; + +async function initialize(shareId) { + info.cookie = ''; + const url = `https://airtable.com/${shareId}`; + + try { + const hreq = await axios + .get(url, { + headers: { + accept: + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'document', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-user': '?1', + 'upgrade-insecure-requests': '1', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + }, + referrerPolicy: 'strict-origin-when-cross-origin', + body: null, + method: 'GET', + }) + .then((response) => { + for (const ck of response.headers['set-cookie']) { + info.cookie += ck.split(';')[0] + '; '; + } + return response.data; + }) + .catch(() => { + throw { + message: + 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + }); + + info.headers = JSON.parse( + hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim() + ); + info.link = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim()); + info.baseInfo = decodeURIComponent(info.link) + .match(/{(.*)}/g)[0] + .split('&') + .reduce((result, el) => { + try { + return Object.assign( + result, + JSON.parse(el.includes('=') ? el.split('=')[1] : el) + ); + } catch (e) { + if (el.includes('=')) { + return Object.assign(result, { + [el.split('=')[0]]: el.split('=')[1], + }); + } + } + }, {}); + info.baseId = info.baseInfo.applicationId; + info.initialized = true; + } catch (e) { + console.log(e); + info.initialized = false; + if (e.message) { + throw e; + } else { + throw { + message: + 'Error processing Shared Base :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + } + } +} + +async function read() { + if (info.initialized) { + const resreq = await axios('https://airtable.com' + info.link, { + headers: { + accept: '*/*', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + 'x-time-zone': 'Europe/Berlin', + cookie: info.cookie, + ...info.headers, + }, + referrerPolicy: 'no-referrer', + body: null, + method: 'GET', + }) + .then((response) => { + return response.data; + }) + .catch(() => { + throw { + message: + 'Error Reading :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + }); + + return { + schema: resreq.data, + baseId: info.baseId, + baseInfo: info.baseInfo, + }; + } else { + throw { + message: 'Error Initializing :: please try again !!', + }; + } +} + +async function readView(viewId) { + if (info.initialized) { + const resreq = await axios( + `https://airtable.com/v0.3/view/${viewId}/readData?` + + `stringifiedObjectParams=${encodeURIComponent('{}')}&requestId=${ + info.baseInfo.requestId + }&accessPolicy=${encodeURIComponent( + JSON.stringify({ + allowedActions: info.baseInfo.allowedActions, + shareId: info.baseInfo.shareId, + applicationId: info.baseInfo.applicationId, + generationNumber: info.baseInfo.generationNumber, + expires: info.baseInfo.expires, + signature: info.baseInfo.signature, + }) + )}`, + { + headers: { + accept: '*/*', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + 'x-time-zone': 'Europe/Berlin', + cookie: info.cookie, + ...info.headers, + }, + referrerPolicy: 'no-referrer', + body: null, + method: 'GET', + } + ) + .then((response) => { + return response.data; + }) + .catch(() => { + throw { + message: + 'Error Reading View :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + }); + return { view: resreq.data }; + } else { + throw { + message: 'Error Initializing :: please try again !!', + }; + } +} + +async function readTemplate(templateId) { + if (!info.initialized) { + await initialize('shrO8aYf3ybwSdDKn'); + } + const resreq = await axios( + `https://www.airtable.com/v0.3/exploreApplications/${templateId}`, + { + headers: { + accept: '*/*', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + 'x-time-zone': 'Europe/Berlin', + cookie: info.cookie, + ...info.headers, + }, + referrer: 'https://www.airtable.com/', + referrerPolicy: 'same-origin', + body: null, + method: 'GET', + mode: 'cors', + credentials: 'include', + } + ) + .then((response) => { + return response.data; + }) + .catch(() => { + throw { + message: + 'Error Fetching :: Ensure www.airtable.com/templates/featured/ is accessible.', + }; + }); + return { template: resreq }; +} + +function unicodeToChar(text) { + return text.replace(/\\u[\dA-F]{4}/gi, function (match) { + return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16)); + }); +} + +export default { + initialize, + read, + readView, + readTemplate, +}; diff --git a/packages/nocodb/src/lib/services/syncService/helpers/job.ts b/packages/nocodb/src/lib/services/syncService/helpers/job.ts new file mode 100644 index 0000000000..80820cfeaf --- /dev/null +++ b/packages/nocodb/src/lib/services/syncService/helpers/job.ts @@ -0,0 +1,2430 @@ +import { Tele } from 'nc-help'; +import FetchAT from './fetchAT'; +import { UITypes } from 'nocodb-sdk'; +// import * as sMap from './syncMap'; + +import { Api } from 'nocodb-sdk'; + +import Airtable from 'airtable'; +import jsonfile from 'jsonfile'; +import hash from 'object-hash'; +import { promisify } from 'util'; + +import dayjs from 'dayjs'; +import utc from 'dayjs/plugin/utc'; +import tinycolor from 'tinycolor2'; +import { importData, importLTARData } from './readAndProcessData'; + +import EntityMap from './EntityMap'; + +const writeJsonFileAsync = promisify(jsonfile.writeFile); + +dayjs.extend(utc); + +const selectColors = { + // normal + blue: '#cfdfff', + cyan: '#d0f0fd', + teal: '#c2f5e9', + green: '#d1f7c4', + orange: '#fee2d5', + yellow: '#ffeab6', + red: '#ffdce5', + pink: '#ffdaf6', + purple: '#ede2fe', + gray: '#eee', + // medium + blueMedium: '#9cc7ff', + cyanMedium: '#77d1f3', + tealMedium: '#72ddc3', + greenMedium: '#93e088', + orangeMedium: '#ffa981', + yellowMedium: '#ffd66e', + redMedium: '#ff9eb7', + pinkMedium: '#f99de2', + purpleMedium: '#cdb0ff', + grayMedium: '#ccc', + // dark + blueDark: '#2d7ff9', + cyanDark: '#18bfff', + tealDark: '#20d9d2', + greenDark: '#20c933', + orangeDark: '#ff6f2c', + yellowDark: '#fcb400', + redDark: '#f82b60', + pinkDark: '#ff08c2', + purpleDark: '#8b46ff', + grayDark: '#666', + // darker + blueDarker: '#2750ae', + cyanDarker: '#0b76b7', + tealDarker: '#06a09b', + greenDarker: '#338a17', + orangeDarker: '#d74d26', + yellowDarker: '#b87503', + redDarker: '#ba1e45', + pinkDarker: '#b2158b', + purpleDarker: '#6b1cb0', + grayDarker: '#444', +}; + +export default async ( + syncDB: AirtableSyncConfig, + progress: (data: { msg?: string; level?: any }) => void +) => { + const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent'); + await sMapEM.init(); + + const sMap = { + // static mapping records between aTblId && ncId + async addToMappingTbl(aTblId, ncId, ncName, ncParent?) { + await sMapEM.addRow({ aTblId, ncId, ncName, ncParent }); + }, + + // get NcID from airtable ID + async getNcIdFromAtId(aId) { + return (await sMapEM.getRow('aTblId', aId, ['ncId']))?.ncId; + }, + + // get nc Parent from airtable ID + async getNcParentFromAtId(aId) { + return (await sMapEM.getRow('aTblId', aId, ['ncParent']))?.ncParent; + }, + + // get nc-title from airtable ID + async getNcNameFromAtId(aId) { + return (await sMapEM.getRow('aTblId', aId, ['ncName']))?.ncName; + }, + }; + + function logBasic(log) { + progress({ level: 0, msg: log }); + } + + function logDetailed(log) { + if (debugMode) progress({ level: 1, msg: log }); + } + + const perfStats = []; + function recordPerfStart() { + if (!debugMode) return 0; + return Date.now(); + } + function recordPerfStats(start, event) { + if (!debugMode) return; + const duration = Date.now() - start; + perfStats.push({ d: duration, e: event }); + } + + let base, baseId; + const start = Date.now(); + const enableErrorLogs = false; + const generate_migrationStats = true; + const debugMode = false; + let api: Api; + let g_aTblSchema = []; + let ncCreatedProjectSchema: any = {}; + const ncLinkMappingTable: any[] = []; + const nestedLookupTbl: any[] = []; + const nestedRollupTbl: any[] = []; + const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' }; + const storeLinks = false; + const ncLinkDataStore: any = {}; + const insertedAssocRef: any = {}; + + const atNcAliasRef: { + [ncTableId: string]: { + [ncTitle: string]: string; + }; + } = {}; + + const uniqueTableNameGen = getUniqueNameGenerator('sheet'); + + // run time counter (statistics) + const rtc = { + sort: 0, + filter: 0, + view: { + total: 0, + grid: 0, + gallery: 0, + form: 0, + }, + fetchAt: { + count: 0, + time: 0, + }, + migrationSkipLog: { + count: 0, + log: [], + }, + data: { + records: 0, + nestedLinks: 0, + }, + }; + + function updateMigrationSkipLog(tbl, col, type, reason?) { + rtc.migrationSkipLog.count++; + rtc.migrationSkipLog.log.push( + `tn[${tbl}] cn[${col}] type[${type}] :: ${reason}` + ); + } + + // mapping table + // + + async function getAirtableSchema(sDB) { + const start = Date.now(); + + if (!sDB.shareId) + throw { + message: + 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + + if (sDB.shareId.startsWith('exp')) { + const template = await FetchAT.readTemplate(sDB.shareId); + await FetchAT.initialize(template.template.exploreApplication.shareId); + } else { + await FetchAT.initialize(sDB.shareId); + } + const ft = await FetchAT.read(); + const duration = Date.now() - start; + rtc.fetchAt.count++; + rtc.fetchAt.time += duration; + + if (!ft.baseId) { + throw { + message: + 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + } + + const file = ft.schema; + baseId = ft.baseId; + base = new Airtable({ apiKey: sDB.apiKey }).base(baseId); + // store copy of airtable schema globally + g_aTblSchema = file.tableSchemas; + + if (debugMode) + await writeJsonFileAsync('aTblSchema.json', ft, { spaces: 2 }); + + return file; + } + + async function getViewData(viewId) { + const start = Date.now(); + const ft = await FetchAT.readView(viewId); + const duration = Date.now() - start; + rtc.fetchAt.count++; + rtc.fetchAt.time += duration; + + if (debugMode) + await writeJsonFileAsync(`${viewId}.json`, ft, { spaces: 2 }); + return ft.view; + } + + function getRootDbType() { + return ncCreatedProjectSchema?.bases.find((el) => el.id === syncDB.baseId) + ?.type; + } + + // base mapping table + const aTblNcTypeMap = { + foreignKey: UITypes.LinkToAnotherRecord, + text: UITypes.SingleLineText, + multilineText: UITypes.LongText, + richText: UITypes.LongText, + multipleAttachment: UITypes.Attachment, + checkbox: UITypes.Checkbox, + multiSelect: UITypes.MultiSelect, + select: UITypes.SingleSelect, + collaborator: UITypes.Collaborator, + multiCollaborator: UITypes.Collaborator, + date: UITypes.Date, + phone: UITypes.PhoneNumber, + number: UITypes.Decimal, + rating: UITypes.Rating, + formula: UITypes.Formula, + rollup: UITypes.Rollup, + count: UITypes.Count, + lookup: UITypes.Lookup, + autoNumber: UITypes.AutoNumber, + barcode: UITypes.SingleLineText, + button: UITypes.Button, + }; + + //----------------------------------------------------------------------------- + // aTbl helper routines + // + + function nc_sanitizeName(name) { + // replace all special characters by _ + return name.replace(/\W+/g, '_').trim(); + } + + function nc_getSanitizedColumnName(table, name) { + let col_name = nc_sanitizeName(name); + + // truncate to 60 chars if character if exceeds above 60 + col_name = col_name?.slice(0, 60); + + // for knex, replace . with _ + const col_alias = name.trim().replace(/\./g, '_'); + + // check if already a column exists with same name? + const duplicateTitle = table.columns.find( + (x) => x.title?.toLowerCase() === col_alias?.toLowerCase() + ); + const duplicateColumn = table.columns.find( + (x) => x.column_name?.toLowerCase() === col_name?.toLowerCase() + ); + if (duplicateTitle) { + if (enableErrorLogs) console.log(`## Duplicate title ${col_alias}`); + } + + return { + // kludge: error observed in Nc with space around column-name + title: col_alias + (duplicateTitle ? '_2' : ''), + column_name: col_name + (duplicateColumn ? '_2' : ''), + }; + } + + // aTbl: retrieve table name from table ID + // + // @ts-ignore + function aTbl_getTableName(tblId) { + const sheetObj = g_aTblSchema.find((tbl) => tbl.id === tblId); + return { + tn: sheetObj.name, + }; + } + + const ncSchema = { + tables: [], + tablesById: {}, + }; + + // aTbl: retrieve column name from column ID + // + function aTbl_getColumnName(colId): any { + for (let i = 0; i < g_aTblSchema.length; i++) { + const sheetObj = g_aTblSchema[i]; + const column = sheetObj.columns.find((col) => col.id === colId); + if (column !== undefined) + return { + tn: sheetObj.name, + cn: column.name, + }; + } + } + + // nc dump schema + // + // @ts-ignore + async function nc_DumpTableSchema() { + console.log('['); + const ncTblList = await api.base.tableList( + ncCreatedProjectSchema.id, + syncDB.baseId + ); + for (let i = 0; i < ncTblList.list.length; i++) { + const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + console.log(JSON.stringify(ncTbl, null, 2)); + console.log(','); + } + console.log(']'); + } + + // retrieve nc column schema from using aTbl field ID as reference + // + async function nc_getColumnSchema(aTblFieldId) { + // let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); + // let aTblField = aTbl_getColumnName(aTblFieldId); + // let ncTblId = ncTblList.list.filter(x => x.title === aTblField.tn)[0].id; + // let ncTbl = await api.dbTable.read(ncTblId); + // let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn); + // return ncCol; + + const ncTblId = await sMap.getNcParentFromAtId(aTblFieldId); + const ncColId = await sMap.getNcIdFromAtId(aTblFieldId); + + // not migrated column, skip + if (ncColId === undefined || ncTblId === undefined) return 0; + + return ncSchema.tablesById[ncTblId].columns.find((x) => x.id === ncColId); + } + + // retrieve nc table schema using table name + // optimize: create a look-up table & re-use information + // + async function nc_getTableSchema(tableName) { + // let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); + // let ncTblId = ncTblList.list.filter(x => x.title === tableName)[0].id; + // let ncTbl = await api.dbTable.read(ncTblId); + // return ncTbl; + + return ncSchema.tables.find((x) => x.title === tableName); + } + + // delete project if already exists + async function init({ + projectName, + }: { + projectName?: string; + projectId?: string; + }) { + // delete 'sample' project if already exists + const x = await api.project.list(); + + const sampleProj = x.list.find((a) => a.title === projectName); + if (sampleProj) { + await api.project.delete(sampleProj.id); + } + logDetailed('Init'); + } + + // map UIDT + // + function getNocoType(col) { + // start with default map + let ncType = aTblNcTypeMap[col.type]; + + // types email & url are marked as text + // types currency & percent, duration are marked as number + // types createTime & modifiedTime are marked as formula + + switch (col.type) { + case 'text': + if (col.typeOptions?.validatorName === 'email') ncType = UITypes.Email; + else if (col.typeOptions?.validatorName === 'url') ncType = UITypes.URL; + break; + + case 'number': + // kludge: currency validation error with decimal places + if (col.typeOptions?.format === 'percentV2') ncType = UITypes.Percent; + else if (col.typeOptions?.format === 'duration') + ncType = UITypes.Duration; + else if (col.typeOptions?.format === 'currency') + ncType = UITypes.Currency; + else if (col.typeOptions?.precision > 0) ncType = UITypes.Decimal; + break; + + case 'formula': + if (col.typeOptions?.formulaTextParsed === 'CREATED_TIME()') + ncType = UITypes.DateTime; + else if (col.typeOptions?.formulaTextParsed === 'LAST_MODIFIED_TIME()') + ncType = UITypes.DateTime; + break; + + case 'computation': + if (col.typeOptions?.resultType === 'collaborator') + ncType = UITypes.Collaborator; + break; + + case 'date': + if (col.typeOptions?.isDateTime) ncType = UITypes.DateTime; + break; + + // case 'barcode': + // case 'button': + // ncType = UITypes.SingleLineText; + // break; + } + + return ncType; + } + + // retrieve additional options associated with selected data types + // + async function getNocoTypeOptions(col: any): Promise { + switch (col.type) { + case 'select': + case 'multiSelect': { + // prepare options list in CSV format + // note: NC doesn't allow comma's in options + // + const options = []; + let order = 1; + for (const [, value] of Object.entries(col.typeOptions.choices)) { + // replace commas with dot for multiselect + if (col.type === 'multiSelect') { + (value as any).name = (value as any).name.replace(/,/g, '.'); + } + // we don't allow empty records, placeholder instead + if ((value as any).name === '') { + (value as any).name = 'nc_empty'; + } + // enumerate duplicates (we don't allow them) + // TODO fix record mapping (this causes every record to map first option, we can't handle them using data api as they don't provide option id within data we might instead get the correct mapping from schema file ) + let dupNo = 1; + const defaultName = (value as any).name; + while ( + options.find( + (el) => + el.title.toLowerCase() === (value as any).name.toLowerCase() + ) + ) { + (value as any).name = `${defaultName}_${dupNo++}`; + } + options.push({ + order: order++, + title: (value as any).name, + color: selectColors[(value as any).color] + ? selectColors[(value as any).color] + : tinycolor.random().toHexString(), + }); + + await sMap.addToMappingTbl( + (value as any).id, + undefined, + (value as any).name + ); + } + return { type: col.type, data: options }; + } + default: + return { type: undefined }; + } + } + + // convert to Nc schema (basic, excluding relations) + // + async function tablesPrepare(tblSchema: any[]) { + const tables: any[] = []; + + for (let i = 0; i < tblSchema.length; ++i) { + const table: any = {}; + + if (syncDB.options.syncViews) { + rtc.view.total += tblSchema[i].views.reduce( + (acc, cur) => + ['grid', 'form', 'gallery'].includes(cur.type) ? ++acc : acc, + 0 + ); + } else { + rtc.view.total = tblSchema.length; + } + + // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id; + table.title = tblSchema[i].name; + let sanitizedName = nc_sanitizeName(tblSchema[i].name); + + // truncate to 50 chars if character if exceeds above 50 + // upto 64 should be fine but we are keeping it to 50 since + // meta project adds prefix as well + sanitizedName = sanitizedName?.slice(0, 50); + + // check for duplicate and populate a unique name if already exist + table.table_name = uniqueTableNameGen(sanitizedName); + + const uniqueColNameGen = getUniqueNameGenerator('field'); + table.columns = []; + const sysColumns = [ + { + title: ncSysFields.id, + column_name: ncSysFields.id, + uidt: UITypes.ID, + meta: { + ag: 'nc', + }, + }, + { + title: ncSysFields.hash, + column_name: ncSysFields.hash, + uidt: UITypes.SingleLineText, + system: true, + }, + ]; + + for (let j = 0; j < tblSchema[i].columns.length; j++) { + const col = tblSchema[i].columns[j]; + + // skip link, lookup, rollup fields in this iteration + if (['foreignKey', 'lookup', 'rollup'].includes(col.type)) { + continue; + } + + // base column schema + const ncName: any = nc_getSanitizedColumnName(table, col.name); + const ncCol: any = { + // Enable to use aTbl identifiers as is: id: col.id, + title: ncName.title, + column_name: uniqueColNameGen(ncName.column_name), + uidt: getNocoType(col), + }; + + // not supported datatype: pure formula field + // allow formula based computed fields (created time/ modified time to go through) + if (ncCol.uidt === UITypes.Formula) { + updateMigrationSkipLog( + tblSchema[i].name, + ncName.title, + col.type, + 'column type not supported' + ); + continue; + } + + // populate cdf (column default value) if configured + // if (col?.default) { + // if (typeof col.default === 'string') + // ncCol.cdf = `'${col.default.replace?.(/'/g, "\\'")}'`; + // else ncCol.cdf = col.default; + // } + + // change from default 'tinytext' as airtable allows more than 255 characters + // for single line text column type + if (col.type === 'text') ncCol.dt = 'text'; + + // #fix-2363-decimal-out-of-range + if (['sqlite3', 'mysql2'].includes(getRootDbType())) { + if (ncCol.uidt === UITypes.Decimal) { + ncCol.dt = 'double'; + ncCol.dtxp = 22; + ncCol.dtxs = '2'; + } + } + + // additional column parameters when applicable + const colOptions = await getNocoTypeOptions(col); + + switch (colOptions.type) { + case 'select': + case 'multiSelect': + ncCol.colOptions = { + options: [...colOptions.data], + }; + + if (['mysql', 'mysql2'].includes(getRootDbType())) { + // if options are empty, configure '' as an option + ncCol.dtxp = + colOptions.data + .map((el) => `'${el.title.replace(/'/gi, "''")}'`) + .join(',') || "''"; + } + + break; + case undefined: + break; + } + table.columns.push(ncCol); + } + table.columns.push(sysColumns[0]); + table.columns.push(sysColumns[1]); + + tables.push(table); + } + return tables; + } + + async function nocoCreateBaseSchema(aTblSchema) { + // base schema preparation: exclude + const tables: any[] = await tablesPrepare(aTblSchema); + + // for each table schema, create nc table + for (let idx = 0; idx < tables.length; idx++) { + logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`); + + logDetailed(`NC API: base.tableCreate ${tables[idx].title}`); + + let _perfStart = recordPerfStart(); + const table: any = await api.base.tableCreate( + ncCreatedProjectSchema.id, + syncDB.baseId, + tables[idx] + ); + recordPerfStats(_perfStart, 'dbTable.create'); + + updateNcTblSchema(table); + + // update mapping table + await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title); + for (let colIdx = 0; colIdx < table.columns.length; colIdx++) { + const aId = aTblSchema[idx].columns.find( + (x) => + x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title + )?.id; + if (aId) + await sMap.addToMappingTbl( + aId, + table.columns[colIdx].id, + table.columns[colIdx].title, + table.id + ); + } + + // update default view name- to match it to airtable view name + logDetailed(`NC API: dbView.list ${table.id}`); + _perfStart = recordPerfStart(); + const view = await api.dbView.list(table.id); + recordPerfStats(_perfStart, 'dbView.list'); + + const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid'); + logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); + _perfStart = recordPerfStart(); + await api.dbView.update(view.list[0].id, { + title: aTbl_grid.name, + }); + recordPerfStats(_perfStart, 'dbView.update'); + + await updateNcTblSchemaById(table.id); + + await sMap.addToMappingTbl( + aTbl_grid.id, + table.views[0].id, + aTbl_grid.name, + table.id + ); + } + + // debug + // console.log(JSON.stringify(tables, null, 2)); + return tables; + } + + async function nocoCreateLinkToAnotherRecord(aTblSchema) { + // Link to another RECORD + for (let idx = 0; idx < aTblSchema.length; idx++) { + const aTblLinkColumns = aTblSchema[idx].columns.filter( + (x) => x.type === 'foreignKey' + ); + + // Link columns exist + // + if (aTblLinkColumns.length) { + for (let i = 0; i < aTblLinkColumns.length; i++) { + logDetailed( + `[${idx + 1}/${aTblSchema.length}] Configuring Links :: [${i + 1}/${ + aTblLinkColumns.length + }] ${aTblSchema[idx].name}` + ); + + // for self links, there is no symmetric column + { + const src = aTbl_getColumnName(aTblLinkColumns[i].id); + const dst = aTbl_getColumnName( + aTblLinkColumns[i].typeOptions?.symmetricColumnId + ); + logDetailed( + `LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst?.tn}:${dst?.cn}` + ); + } + + // check if link already established? + if (!nc_isLinkExists(aTblLinkColumns[i].id)) { + // parent table ID + // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + + // find child table name from symmetric column ID specified + // self link, symmetricColumnId field will be undefined + const childTable = aTbl_getColumnName( + aTblLinkColumns[i].typeOptions?.symmetricColumnId + ); + + // retrieve child table ID (nc) from table name + let childTableId = srcTableId; + if (childTable) { + childTableId = (await nc_getTableSchema(childTable.tn)).id; + } + + // check if already a column exists with this name? + let _perfStart = recordPerfStart(); + const srcTbl: any = await api.dbTable.read(srcTableId); + recordPerfStats(_perfStart, 'dbTable.read'); + + // create link + const ncName = nc_getSanitizedColumnName( + srcTbl, + aTblLinkColumns[i].name + ); + + // LTAR alias ref to AT + atNcAliasRef[srcTbl.id] = atNcAliasRef[srcTbl.id] || {}; + atNcAliasRef[srcTbl.id][ncName.title] = aTblLinkColumns[i].name; + + logDetailed( + `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` + ); + _perfStart = recordPerfStart(); + const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + uidt: UITypes.LinkToAnotherRecord, + title: ncName.title, + column_name: ncName.column_name, + parentId: srcTableId, + childId: childTableId, + type: 'mm', + // aTblLinkColumns[i].typeOptions.relationship === 'many' + // ? 'mm' + // : 'hm' + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === ncName.title + )?.id; + await sMap.addToMappingTbl( + aTblLinkColumns[i].id, + ncId, + ncName.title, + ncTbl.id + ); + + // store link information in separate table + // this information will be helpful in identifying relation pair + const link = { + nc: { + title: ncName.title, + parentId: srcTableId, + childId: childTableId, + type: 'mm', + }, + aTbl: { + tblId: aTblSchema[idx].id, + ...aTblLinkColumns[i], + }, + }; + + ncLinkMappingTable.push(link); + } else { + // if link already exists, we need to change name of linked column + // to what is represented in airtable + + // 1. extract associated link information from link table + // 2. retrieve parent table information (source) + // 3. using foreign parent & child column ID, find associated mapping in child table + // 4. update column name + const x = ncLinkMappingTable.findIndex( + (x) => + x.aTbl.tblId === + aTblLinkColumns[i].typeOptions.foreignTableId && + x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId + ); + + let _perfStart = recordPerfStart(); + const childTblSchema: any = await api.dbTable.read( + ncLinkMappingTable[x].nc.childId + ); + recordPerfStats(_perfStart, 'dbTable.read'); + + _perfStart = recordPerfStart(); + const parentTblSchema: any = await api.dbTable.read( + ncLinkMappingTable[x].nc.parentId + ); + recordPerfStats(_perfStart, 'dbTable.read'); + + // fix me + // let childTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.childId] + // let parentTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.parentId] + + let parentLinkColumn = parentTblSchema.columns.find( + (col) => col.title === ncLinkMappingTable[x].nc.title + ); + + if (parentLinkColumn === undefined) { + updateMigrationSkipLog( + parentTblSchema?.title, + ncLinkMappingTable[x].nc.title, + UITypes.LinkToAnotherRecord, + 'Link error' + ); + continue; + } + + // hack // fix me + if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') { + parentLinkColumn = parentTblSchema.columns.find( + (col) => col.title === ncLinkMappingTable[x].nc.title + '_2' + ); + } + + let childLinkColumn: any = {}; + + if (parentLinkColumn.colOptions.type == 'hm') { + // for hm: + // mapping between child & parent column id is direct + // + childLinkColumn = childTblSchema.columns.find( + (col) => + col.uidt === UITypes.LinkToAnotherRecord && + col.colOptions.fk_child_column_id === + parentLinkColumn.colOptions.fk_child_column_id && + col.colOptions.fk_parent_column_id === + parentLinkColumn.colOptions.fk_parent_column_id + ); + } else { + // for mm: + // mapping between child & parent column id is inverted + // + childLinkColumn = childTblSchema.columns.find( + (col) => + col.uidt === UITypes.LinkToAnotherRecord && + col.colOptions.fk_child_column_id === + parentLinkColumn.colOptions.fk_parent_column_id && + col.colOptions.fk_parent_column_id === + parentLinkColumn.colOptions.fk_child_column_id && + col.colOptions.fk_mm_model_id === + parentLinkColumn.colOptions.fk_mm_model_id + ); + } + + // check if already a column exists with this name? + const duplicate = childTblSchema.columns.find( + (x) => x.title === aTblLinkColumns[i].name + ); + const suffix = duplicate ? '_2' : ''; + if (duplicate) + if (enableErrorLogs) + console.log(`## Duplicate ${aTblLinkColumns[i].name}`); + + // rename + // note that: current rename API requires us to send all parameters, + // not just title being renamed + const ncName = nc_getSanitizedColumnName( + childTblSchema, + aTblLinkColumns[i].name + ); + + logDetailed( + `NC API: dbTableColumn.update rename symmetric column ${ncName.title}` + ); + _perfStart = recordPerfStart(); + const ncTbl: any = await api.dbTableColumn.update( + childLinkColumn.id, + { + ...childLinkColumn, + title: ncName.title, + column_name: ncName.column_name, + } + ); + recordPerfStats(_perfStart, 'dbTableColumn.update'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === aTblLinkColumns[i].name + suffix + )?.id; + await sMap.addToMappingTbl( + aTblLinkColumns[i].id, + ncId, + aTblLinkColumns[i].name + suffix, + ncTbl.id + ); + + // console.log(res.columns.find(x => x.title === aTblLinkColumns[i].name)) + } + } + } + } + } + + async function nocoCreateLookups(aTblSchema) { + // LookUps + for (let idx = 0; idx < aTblSchema.length; idx++) { + const aTblColumns = aTblSchema[idx].columns.filter( + (x) => x.type === 'lookup' + ); + + // parent table ID + // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + if (aTblColumns.length) { + // Lookup + for (let i = 0; i < aTblColumns.length; i++) { + logDetailed( + `[${idx + 1}/${aTblSchema.length}] Configuring Lookup :: [${ + i + 1 + }/${aTblColumns.length}] ${aTblSchema[idx].name}` + ); + + // something is not right, skip + if ( + aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + if (enableErrorLogs) + console.log(`## Invalid column IDs mapped; skip`); + + updateMigrationSkipLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + 'invalid column ID in dependency list' + ); + continue; + } + + const ncRelationColumnId = await sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.relationColumnId + ); + const ncLookupColumnId = await sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + + if ( + ncLookupColumnId === undefined || + ncRelationColumnId === undefined + ) { + aTblColumns[i]['srcTableId'] = srcTableId; + nestedLookupTbl.push(aTblColumns[i]); + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + aTblColumns[i].name + ); + + logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); + const _perfStart = recordPerfStart(); + const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === aTblColumns[i].name + )?.id; + await sMap.addToMappingTbl( + aTblColumns[i].id, + ncId, + aTblColumns[i].name, + ncTbl.id + ); + } + } + } + + let level = 2; + let nestedCnt = 0; + while (nestedLookupTbl.length) { + // if nothing has changed from previous iteration, skip rest + if (nestedCnt === nestedLookupTbl.length) { + for (let i = 0; i < nestedLookupTbl.length; i++) { + const fTblField = + nestedLookupTbl[i].typeOptions.foreignTableRollupColumnId; + const name = aTbl_getColumnName(fTblField); + updateMigrationSkipLog( + ncSchema.tablesById[nestedLookupTbl[i].srcTableId]?.title, + nestedLookupTbl[i].name, + nestedLookupTbl[i].type, + `foreign table field not found [${name.tn}/${name.cn}]` + ); + } + if (enableErrorLogs) + console.log( + `## Failed to configure ${nestedLookupTbl.length} lookups` + ); + break; + } + + // Nested lookup + nestedCnt = nestedLookupTbl.length; + for (let i = 0; i < nestedLookupTbl.length; i++) { + const srcTableId = nestedLookupTbl[0].srcTableId; + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + const ncRelationColumnId = await sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.relationColumnId + ); + const ncLookupColumnId = await sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId + ); + + if ( + ncLookupColumnId === undefined || + ncRelationColumnId === undefined + ) { + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + nestedLookupTbl[0].name + ); + + logDetailed( + `Configuring Nested Lookup: Level-${level} [${i + 1}/${nestedCnt} ${ + ncName.title + }]` + ); + + logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); + const _perfStart = recordPerfStart(); + const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === nestedLookupTbl[0].name + )?.id; + await sMap.addToMappingTbl( + nestedLookupTbl[0].id, + ncId, + nestedLookupTbl[0].name, + ncTbl.id + ); + + // remove entry + nestedLookupTbl.splice(0, 1); + } + level++; + } + } + + function getRollupNcFunction(aTblFunction) { + const fn = aTblFunction.split('(')[0]; + const aTbl_ncRollUp = { + AND: '', + ARRAYCOMPACT: '', + ARRAYJOIN: '', + ARRAYUNIQUE: '', + AVERAGE: 'average', + CONCATENATE: '', + COUNT: 'count', + COUNTA: '', + COUNTALL: '', + MAX: 'max', + MIN: 'min', + OR: '', + SUM: 'sum', + XOR: '', + }; + return aTbl_ncRollUp[fn]; + } + + //@ts-ignore + async function nocoCreateRollup(aTblSchema) { + // Rollup + for (let idx = 0; idx < aTblSchema.length; idx++) { + const aTblColumns = aTblSchema[idx].columns.filter( + (x) => x.type === 'rollup' + ); + + // parent table ID + // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + if (aTblColumns.length) { + // rollup exist + for (let i = 0; i < aTblColumns.length; i++) { + logDetailed( + `[${idx + 1}/${aTblSchema.length}] Configuring Rollup :: [${ + i + 1 + }/${aTblColumns.length}] ${aTblSchema[idx].name}` + ); + + // fetch associated rollup function + // skip column creation if supported rollup function does not exist + const ncRollupFn = getRollupNcFunction( + aTblColumns[i].typeOptions.formulaTextParsed + ); + // const ncRollupFn = ''; + + if (ncRollupFn === '' || ncRollupFn === undefined) { + updateMigrationSkipLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + `rollup function ${aTblColumns[i].typeOptions.formulaTextParsed} not supported` + ); + continue; + } + + // something is not right, skip + if ( + aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + if (enableErrorLogs) + console.log(`## Invalid column IDs mapped; skip`); + + updateMigrationSkipLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + 'invalid column ID in dependency list' + ); + continue; + } + + const ncRelationColumnId = await sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.relationColumnId + ); + const ncRollupColumnId = await sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + + if (ncRollupColumnId === undefined) { + aTblColumns[i]['srcTableId'] = srcTableId; + nestedRollupTbl.push(aTblColumns[i]); + continue; + } + + // skip, if rollup column was pointing to another virtual column + const ncColSchema = await nc_getColumnSchema( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + if ( + ncColSchema?.uidt === UITypes.Formula || + ncColSchema?.uidt === UITypes.Lookup || + ncColSchema?.uidt === UITypes.Rollup || + ncColSchema?.uidt === UITypes.Checkbox + ) { + updateMigrationSkipLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + 'rollup referring to a column type not supported currently' + ); + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + aTblColumns[i].name + ); + + logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); + const _perfStart = recordPerfStart(); + const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + uidt: UITypes.Rollup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_rollup_column_id: ncRollupColumnId, + rollup_function: ncRollupFn, + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === aTblColumns[i].name + )?.id; + await sMap.addToMappingTbl( + aTblColumns[i].id, + ncId, + aTblColumns[i].name, + ncTbl.id + ); + } + } + } + logDetailed(`Nested rollup: ${nestedRollupTbl.length}`); + } + + //@ts-ignore + async function nocoLookupForRollup() { + const nestedCnt = nestedLookupTbl.length; + for (let i = 0; i < nestedLookupTbl.length; i++) { + const srcTableId = nestedLookupTbl[0].srcTableId; + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + const ncRelationColumnId = await sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.relationColumnId + ); + const ncLookupColumnId = await sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId + ); + + if (ncLookupColumnId === undefined || ncRelationColumnId === undefined) { + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + nestedLookupTbl[0].name + ); + + logDetailed( + `Configuring Lookup over Rollup :: [${i + 1}/${nestedCnt}] ${ + ncName.title + }` + ); + + logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); + const _perfStart = recordPerfStart(); + const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); + + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + (x) => x.title === nestedLookupTbl[0].name + )?.id; + await sMap.addToMappingTbl( + nestedLookupTbl[0].id, + ncId, + nestedLookupTbl[0].name, + ncTbl.id + ); + + // remove entry + nestedLookupTbl.splice(0, 1); + } + } + + async function nocoSetPrimary(aTblSchema) { + for (let idx = 0; idx < aTblSchema.length; idx++) { + logDetailed( + `[${idx + 1}/${aTblSchema.length}] Configuring Display value : ${ + aTblSchema[idx].name + }` + ); + + const pColId = aTblSchema[idx].primaryColumnId; + const ncColId = await sMap.getNcIdFromAtId(pColId); + + // skip primary column configuration if we field not migrated + if (ncColId) { + logDetailed(`NC API: dbTableColumn.primaryColumnSet`); + const _perfStart = recordPerfStart(); + await api.dbTableColumn.primaryColumnSet(ncColId); + recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet'); + + // update schema + const ncTblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + await updateNcTblSchemaById(ncTblId); + } + } + } + + // retrieve nc-view column ID from corresponding nc-column ID + async function nc_getViewColumnId(viewId, viewType, ncColumnId) { + // retrieve view Info + let viewDetails; + + const _perfStart = recordPerfStart(); + if (viewType === 'form') { + viewDetails = (await api.dbView.formRead(viewId)).columns; + recordPerfStats(_perfStart, 'dbView.formRead'); + } else if (viewType === 'gallery') { + viewDetails = (await api.dbView.galleryRead(viewId)).columns; + recordPerfStats(_perfStart, 'dbView.galleryRead'); + } else { + viewDetails = await api.dbView.gridColumnsList(viewId); + recordPerfStats(_perfStart, 'dbView.gridColumnsList'); + } + + return viewDetails.find((x) => x.fk_column_id === ncColumnId)?.id; + } + + ////////// Data processing + + async function nocoBaseDataProcessing_v2(sDB, table, record) { + const recordHash = hash(record); + const rec = { ...record.fields }; + + // kludge - + // trim spaces on either side of column name + // leads to error in NocoDB + Object.keys(rec).forEach((key) => { + const replacedKey = key.trim().replace(/\./g, '_'); + if (key !== replacedKey) { + rec[replacedKey] = rec[key]; + delete rec[key]; + } + }); + + // post-processing on the record + for (const [key, value] of Object.entries(rec as { [key: string]: any })) { + // retrieve datatype + const dt = table.columns.find((x) => x.title === key)?.uidt; + + // always process LTAR, Lookup, and Rollup columns as we delete the key after processing + if ( + !value && + dt !== UITypes.LinkToAnotherRecord && + dt !== UITypes.Lookup && + dt !== UITypes.Rollup + ) { + rec[key] = null; + continue; + } + + switch (dt) { + // https://www.npmjs.com/package/validator + // default value: digits_after_decimal: [2] + // if currency, set decimal place to 2 + // + case UITypes.Currency: + rec[key] = (+value).toFixed(2); + break; + + // we will pick up LTAR once all table data's are in place + case UITypes.LinkToAnotherRecord: + if (storeLinks) { + if (ncLinkDataStore[table.title][record.id] === undefined) + ncLinkDataStore[table.title][record.id] = { + id: record.id, + fields: {}, + }; + ncLinkDataStore[table.title][record.id]['fields'][key] = value; + } + delete rec[key]; + break; + + // these will be automatically populated depending on schema configuration + case UITypes.Lookup: + case UITypes.Rollup: + delete rec[key]; + break; + + case UITypes.Collaborator: + // in case of multi-collaborator, this will be an array + if (Array.isArray(value)) { + let collaborators = ''; + for (let i = 0; i < value.length; i++) { + collaborators += `${value[i]?.name} <${value[i]?.email}>, `; + rec[key] = collaborators; + } + } else rec[key] = `${value?.name} <${value?.email}>`; + break; + + case UITypes.Button: + rec[key] = `${value?.label} <${value?.url}>`; + break; + + case UITypes.DateTime: + case UITypes.CreateTime: + case UITypes.LastModifiedTime: + rec[key] = dayjs(value).utc().format('YYYY-MM-DD HH:mm'); + break; + + case UITypes.Date: + if (/\d{5,}/.test(value)) { + // skip + rec[key] = null; + logBasic(`:: Invalid date ${value}`); + } else { + rec[key] = dayjs(value).utc().format('YYYY-MM-DD'); + } + break; + + case UITypes.SingleSelect: + if (value === '') { + rec[key] = 'nc_empty'; + } + rec[key] = value; + break; + + case UITypes.MultiSelect: + rec[key] = value + ?.map((v) => { + if (v === '') { + return 'nc_empty'; + } + return `${v.replace(/,/g, '.')}`; + }) + .join(','); + break; + + case UITypes.Attachment: + if (!syncDB.options.syncAttachment) rec[key] = null; + else { + let tempArr = []; + + try { + logBasic( + ` :: Retrieving attachment :: ${value + ?.map((a) => a.filename?.split('?')?.[0]) + .join(', ')}` + ); + tempArr = await api.storage.uploadByUrl( + { + path: `noco/${sDB.projectName}/${table.title}/${key}`, + }, + value?.map((attachment) => ({ + fileName: attachment.filename?.split('?')?.[0], + url: attachment.url, + size: attachment.size, + mimetype: attachment.type, + })) + ); + } catch (e) { + console.log(e); + } + + rec[key] = JSON.stringify(tempArr); + } + break; + + case UITypes.SingleLineText: + // Barcode data + if (value?.text) { + rec[key] = value.text; + } + break; + + default: + break; + } + } + + // insert airtable record ID explicitly into each records + rec[ncSysFields.id] = record.id; + rec[ncSysFields.hash] = recordHash; + + return rec; + } + + // @ts-ignore + async function nocoReadDataSelected(projName, table, callback, fields) { + return new Promise((resolve, reject) => { + base(table.title) + .select({ + pageSize: 100, + // maxRecords: 100, + fields: fields, + }) + .eachPage( + async function page(records, fetchNextPage) { + // console.log(JSON.stringify(records, null, 2)); + + // This function (`page`) will get called for each page of records. + // records.forEach(record => callback(table, record)); + logBasic( + `:: ${table.title} / ${fields} : ${ + recordCnt + 1 + } ~ ${(recordCnt += 100)}` + ); + await Promise.all( + records.map((r) => callback(projName, table, r, fields)) + ); + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + function done(err) { + if (err) { + console.error(err); + reject(err); + } + resolve(null); + } + ); + }); + } + + ////////// + + function nc_isLinkExists(airtableFieldId) { + return !!ncLinkMappingTable.find( + (x) => x.aTbl.typeOptions.symmetricColumnId === airtableFieldId + ); + } + + async function nocoCreateProject(projName) { + // create empty project (XC-DB) + logDetailed(`Create Project: ${projName}`); + const _perfStart = recordPerfStart(); + ncCreatedProjectSchema = await api.project.create({ + title: projName, + }); + recordPerfStats(_perfStart, 'project.create'); + } + + async function nocoGetProject(projId) { + // create empty project (XC-DB) + logDetailed(`Getting project meta: ${projId}`); + const _perfStart = recordPerfStart(); + ncCreatedProjectSchema = await api.project.read(projId); + recordPerfStats(_perfStart, 'project.read'); + } + + async function nocoConfigureGalleryView(sDB, aTblSchema) { + if (!sDB.options.syncViews) return; + for (let idx = 0; idx < aTblSchema.length; idx++) { + const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + const galleryViews = aTblSchema[idx].views.filter( + (x) => x.type === 'gallery' + ); + + const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; + rtc.view.gallery += galleryViews.length; + + for (let i = 0; i < galleryViews.length; i++) { + logDetailed(` Axios fetch view-data`); + + // create view + await getViewData(galleryViews[i].id); + const viewName = aTblSchema[idx].views.find( + (x) => x.id === galleryViews[i].id + )?.name; + + logBasic( + `:: [${configuredViews + i + 1}/${rtc.view.total}] Gallery : ${ + aTblSchema[idx].name + } / ${viewName}` + ); + + logDetailed(`NC API dbView.galleryCreate :: ${viewName}`); + const _perfStart = recordPerfStart(); + await api.dbView.galleryCreate(tblId, { title: viewName }); + recordPerfStats(_perfStart, 'dbView.galleryCreate'); + + await updateNcTblSchemaById(tblId); + // syncLog(`[${idx+1}/${aTblSchema.length}][Gallery View][${i+1}/${galleryViews.length}] Create ${viewName}`) + + // await nc_configureFields(g.id, vData, aTblSchema[idx].name, viewName, 'gallery'); + } + } + } + + async function nocoConfigureFormView(sDB, aTblSchema) { + if (!sDB.options.syncViews) return; + for (let idx = 0; idx < aTblSchema.length; idx++) { + const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + const formViews = aTblSchema[idx].views.filter((x) => x.type === 'form'); + + const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; + rtc.view.form += formViews.length; + for (let i = 0; i < formViews.length; i++) { + logDetailed(` Axios fetch view-data`); + + // create view + const vData = await getViewData(formViews[i].id); + const viewName = aTblSchema[idx].views.find( + (x) => x.id === formViews[i].id + )?.name; + + logBasic( + `:: [${configuredViews + i + 1}/${rtc.view.total}] Form : ${ + aTblSchema[idx].name + } / ${viewName}` + ); + + // everything is default + let refreshMode = 'NO_REFRESH'; + let msg = 'Thank you for submitting the form!'; + let desc = ''; + + // response will not include form object if everything is default + // + if (vData.metadata?.form) { + if (vData.metadata.form?.refreshAfterSubmit) + refreshMode = vData.metadata.form.refreshAfterSubmit; + if (vData.metadata.form?.afterSubmitMessage) + msg = vData.metadata.form.afterSubmitMessage; + if (vData.metadata.form?.description) + desc = vData.metadata.form.description; + } + + const formData = { + title: viewName, + heading: viewName, + subheading: desc, + success_msg: msg, + submit_another_form: refreshMode.includes('REFRESH_BUTTON'), + show_blank_form: refreshMode.includes('AUTO_REFRESH'), + }; + + logDetailed(`NC API dbView.formCreate :: ${viewName}`); + const _perfStart = recordPerfStart(); + const f = await api.dbView.formCreate(tblId, formData); + recordPerfStats(_perfStart, 'dbView.formCreate'); + + logDetailed( + `[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${ + formViews.length + }] Create ${viewName}` + ); + + await updateNcTblSchemaById(tblId); + + logDetailed(` Configure show/hide columns`); + await nc_configureFields( + f.id, + vData, + aTblSchema[idx].name, + viewName, + 'form' + ); + } + } + } + + async function nocoConfigureGridView(sDB, aTblSchema) { + for (let idx = 0; idx < aTblSchema.length; idx++) { + const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); + const gridViews = aTblSchema[idx].views.filter((x) => x.type === 'grid'); + + let viewCnt = idx; + if (syncDB.options.syncViews) + viewCnt = rtc.view.grid + rtc.view.gallery + rtc.view.form; + rtc.view.grid += gridViews.length; + + for (let i = 0; i < (sDB.options.syncViews ? gridViews.length : 1); i++) { + logDetailed(` Axios fetch view-data`); + // fetch viewData JSON + const vData = await getViewData(gridViews[i].id); + + // retrieve view name & associated NC-ID + const viewName = aTblSchema[idx].views.find( + (x) => x.id === gridViews[i].id + )?.name; + const _perfStart = recordPerfStart(); + const viewList: any = await api.dbView.list(tblId); + recordPerfStats(_perfStart, 'dbView.list'); + + let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id; + + logBasic( + `:: [${viewCnt + i + 1}/${rtc.view.total}] Grid : ${ + aTblSchema[idx].name + } / ${viewName}` + ); + + // create view (default already created) + if (i > 0) { + logDetailed(`NC API dbView.gridCreate :: ${viewName}`); + const _perfStart = recordPerfStart(); + const viewCreated = await api.dbView.gridCreate(tblId, { + title: viewName, + }); + recordPerfStats(_perfStart, 'dbView.gridCreate'); + + await updateNcTblSchemaById(tblId); + await sMap.addToMappingTbl( + gridViews[i].id, + viewCreated.id, + viewName, + tblId + ); + // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Create ${viewName}`) + ncViewId = viewCreated.id; + } + + // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Hide columns ${viewName}`) + logDetailed(` Configure show/hide columns`); + await nc_configureFields( + ncViewId, + vData, + aTblSchema[idx].name, + viewName, + 'grid' + ); + + // configure filters + if (vData?.filters) { + // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure filters ${viewName}`) + logDetailed(` Configure filter set`); + + // skip filters if nested + if (!vData.filters.filterSet.find((x) => x?.type === 'nested')) { + await nc_configureFilters(ncViewId, vData.filters); + } + } + + // configure sort + if (vData?.lastSortsApplied?.sortSet.length) { + // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure sort ${viewName}`) + logDetailed(` Configure sort set`); + await nc_configureSort(ncViewId, vData.lastSortsApplied); + } + } + } + } + + async function nocoAddUsers(aTblSchema) { + const userRoles = { + owner: 'owner', + create: 'creator', + edit: 'editor', + comment: 'commenter', + read: 'viewer', + none: 'viewer', + }; + const userList = aTblSchema.appBlanket.userInfoById; + const totalUsers = Object.keys(userList).length; + let cnt = 0; + const insertJobs: Promise[] = []; + + for (const [, value] of Object.entries( + userList as { [key: string]: any } + )) { + logDetailed( + `[${++cnt}/${totalUsers}] NC API auth.projectUserAdd :: ${value.email}` + ); + const _perfStart = recordPerfStart(); + insertJobs.push( + api.auth + .projectUserAdd(ncCreatedProjectSchema.id, { + email: value.email, + roles: userRoles[value.permissionLevel], + }) + .catch((e) => + e.response?.data?.msg + ? logBasic(`NOTICE: ${e.response.data.msg}`) + : console.log(e) + ) + ); + recordPerfStats(_perfStart, 'auth.projectUserAdd'); + } + await Promise.all(insertJobs); + } + + function updateNcTblSchema(tblSchema) { + const tblId = tblSchema.id; + + // replace entry from array if already exists + const idx = ncSchema.tables.findIndex((x) => x.id === tblId); + if (idx !== -1) ncSchema.tables.splice(idx, 1); + ncSchema.tables.push(tblSchema); + + // overwrite object if it exists + ncSchema.tablesById[tblId] = tblSchema; + } + + async function updateNcTblSchemaById(tblId) { + const _perfStart = recordPerfStart(); + const ncTbl = await api.dbTable.read(tblId); + recordPerfStats(_perfStart, 'dbTable.read'); + + updateNcTblSchema(ncTbl); + } + + /////////////////////// + + // statistics + // + const migrationStats = []; + + async function generateMigrationStats(aTblSchema) { + const migrationStatsObj = { + table_name: '', + aTbl: { + columns: 0, + links: 0, + lookup: 0, + rollup: 0, + }, + nc: { + columns: 0, + links: 0, + lookup: 0, + rollup: 0, + invalidColumn: 0, + }, + }; + for (let idx = 0; idx < aTblSchema.length; idx++) { + migrationStatsObj.table_name = aTblSchema[idx].name; + + const aTblLinkColumns = aTblSchema[idx].columns.filter( + (x) => x.type === 'foreignKey' + ); + const aTblLookup = aTblSchema[idx].columns.filter( + (x) => x.type === 'lookup' + ); + const aTblRollup = aTblSchema[idx].columns.filter( + (x) => x.type === 'rollup' + ); + + let invalidColumnId = 0; + for (let i = 0; i < aTblLookup.length; i++) { + if ( + aTblLookup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + invalidColumnId++; + } + } + for (let i = 0; i < aTblRollup.length; i++) { + if ( + aTblRollup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + invalidColumnId++; + } + } + + migrationStatsObj.aTbl.columns = aTblSchema[idx].columns.length; + migrationStatsObj.aTbl.links = aTblLinkColumns.length; + migrationStatsObj.aTbl.lookup = aTblLookup.length; + migrationStatsObj.aTbl.rollup = aTblRollup.length; + + const ncTbl = await nc_getTableSchema(aTblSchema[idx].name); + const linkColumn = ncTbl.columns.filter( + (x) => x.uidt === UITypes.LinkToAnotherRecord + ); + const lookup = ncTbl.columns.filter((x) => x.uidt === UITypes.Lookup); + const rollup = ncTbl.columns.filter((x) => x.uidt === UITypes.Rollup); + + // all links hardwired as m2m. m2m generates additional tables per link + // hence link/2 + migrationStatsObj.nc.columns = + ncTbl.columns.length - linkColumn.length / 2; + migrationStatsObj.nc.links = linkColumn.length / 2; + migrationStatsObj.nc.lookup = lookup.length; + migrationStatsObj.nc.rollup = rollup.length; + migrationStatsObj.nc.invalidColumn = invalidColumnId; + + const temp = JSON.parse(JSON.stringify(migrationStatsObj)); + migrationStats.push(temp); + } + + const columnSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.columns; + }, 0); + const linkSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.links; + }, 0); + const lookupSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.lookup; + }, 0); + const rollupSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.rollup; + }, 0); + + logBasic(`Quick Summary:`); + logBasic(`:: Total Tables: ${aTblSchema.length}`); + logBasic(`:: Total Columns: ${columnSum}`); + logBasic(`:: Links: ${linkSum}`); + logBasic(`:: Lookup: ${lookupSum}`); + logBasic(`:: Rollup: ${rollupSum}`); + logBasic(`:: Total Filters: ${rtc.filter}`); + logBasic(`:: Total Sort: ${rtc.sort}`); + logBasic(`:: Total Views: ${rtc.view.total}`); + logBasic(`:: Grid: ${rtc.view.grid}`); + logBasic(`:: Gallery: ${rtc.view.gallery}`); + logBasic(`:: Form: ${rtc.view.form}`); + logBasic(`:: Total Records: ${rtc.data.records}`); + logBasic(`:: Total Nested Links: ${rtc.data.nestedLinks}`); + + const duration = Date.now() - start; + logBasic(`:: Migration time: ${duration}`); + logBasic(`:: Axios fetch count: ${rtc.fetchAt.count}`); + logBasic(`:: Axios fetch time: ${rtc.fetchAt.time}`); + + if (debugMode) { + await writeJsonFileAsync('stats.json', perfStats, { spaces: 2 }); + const perflog = []; + for (let i = 0; i < perfStats.length; i++) { + perflog.push(`${perfStats[i].e}, ${perfStats[i].d}`); + } + await writeJsonFileAsync('stats.csv', perflog, { spaces: 2 }); + await writeJsonFileAsync('skip.txt', rtc.migrationSkipLog.log, { + spaces: 2, + }); + } + + Tele.event({ + event: 'a:airtable-import:success', + data: { + stats: { + migrationTime: duration, + totalTables: aTblSchema.length, + totalColumns: columnSum, + links: linkSum, + lookup: lookupSum, + rollup: rollupSum, + totalFilters: rtc.filter, + totalSort: rtc.sort, + view: { + total: rtc.view.total, + grid: rtc.view.grid, + gallery: rtc.view.gallery, + form: rtc.view.form, + }, + axios: { + count: rtc.fetchAt.count, + time: rtc.fetchAt.time, + }, + totalRecords: rtc.data.records, + nestedLinks: rtc.data.nestedLinks, + }, + }, + }); + } + + ////////////////////////////// + // filters + + const filterMap = { + '=': 'eq', + '!=': 'neq', + '<': 'lt', + '<=': 'lte', + '>': 'gt', + '>=': 'gte', + isEmpty: 'empty', + isNotEmpty: 'notempty', + contains: 'like', + doesNotContain: 'nlike', + isAnyOf: 'anyof', + isNoneOf: 'nanyof', + '|': 'anyof', + '&': 'allof', + }; + + async function nc_configureFilters(viewId, f) { + for (let i = 0; i < f.filterSet.length; i++) { + const filter = f.filterSet[i]; + const colSchema = await nc_getColumnSchema(filter.columnId); + + // column not available; + // one of not migrated column; + if (!colSchema) { + updateMigrationSkipLog( + await sMap.getNcNameFromAtId(viewId), + colSchema.title, + colSchema.uidt, + `filter config skipped; column not migrated` + ); + continue; + } + const columnId = colSchema.id; + const datatype = colSchema.uidt; + const ncFilters = []; + + // console.log(filter) + if (datatype === UITypes.Date || datatype === UITypes.DateTime) { + // skip filters over data datatype + updateMigrationSkipLog( + await sMap.getNcNameFromAtId(viewId), + colSchema.title, + colSchema.uidt, + `filter config skipped; filter over date datatype not supported` + ); + continue; + } + + // single-select & multi-select + else if ( + datatype === UITypes.SingleSelect || + datatype === UITypes.MultiSelect + ) { + if (filter.operator === 'doesNotContain') { + filter.operator = 'isNoneOf'; + } + // if array, break it down to multiple filters + if (Array.isArray(filter.value)) { + const fx = { + fk_column_id: columnId, + logical_op: f.conjunction, + comparison_op: filterMap[filter.operator], + value: ( + await Promise.all( + filter.value.map(async (f) => await sMap.getNcNameFromAtId(f)) + ) + ).join(','), + }; + ncFilters.push(fx); + } + // not array - add as is + else if (filter.value) { + const fx = { + fk_column_id: columnId, + logical_op: f.conjunction, + comparison_op: filterMap[filter.operator], + value: await sMap.getNcNameFromAtId(filter.value), + }; + ncFilters.push(fx); + } + } + + // other data types (number/ text/ long text/ ..) + else if (filter.value) { + const fx = { + fk_column_id: columnId, + logical_op: f.conjunction, + comparison_op: filterMap[filter.operator], + value: filter.value, + }; + ncFilters.push(fx); + } + + // insert filters + for (let i = 0; i < ncFilters.length; i++) { + const _perfStart = recordPerfStart(); + await api.dbTableFilter.create(viewId, { + ...ncFilters[i], + }); + recordPerfStats(_perfStart, 'dbTableFilter.create'); + + rtc.filter++; + } + } + } + + async function nc_configureSort(viewId, s) { + for (let i = 0; i < s.sortSet.length; i++) { + const columnId = (await nc_getColumnSchema(s.sortSet[i].columnId))?.id; + + if (columnId) { + const _perfStart = recordPerfStart(); + await api.dbTableSort.create(viewId, { + fk_column_id: columnId, + direction: s.sortSet[i].ascending ? 'asc' : 'dsc', + }); + recordPerfStats(_perfStart, 'dbTableSort.create'); + } + rtc.sort++; + } + } + + async function nc_configureFields(_viewId, _c, tblName, viewName, viewType?) { + // force hide PK column + const hiddenColumns = [ncSysFields.id, ncSysFields.hash]; + const c = _c.columnOrder; + + // column order corrections + // retrieve table schema + const ncTbl = await nc_getTableSchema(tblName); + // retrieve view ID + const viewId = ncTbl.views.find((x) => x.title === viewName).id; + let viewDetails; + + const _perfStart = recordPerfStart(); + if (viewType === 'form') { + viewDetails = (await api.dbView.formRead(viewId)).columns; + recordPerfStats(_perfStart, 'dbView.formRead'); + } else if (viewType === 'gallery') { + viewDetails = (await api.dbView.galleryRead(viewId)).columns; + recordPerfStats(_perfStart, 'dbView.galleryRead'); + } else { + viewDetails = await api.dbView.gridColumnsList(viewId); + recordPerfStats(_perfStart, 'dbView.gridColumnsList'); + } + + // nc-specific columns; default hide. + for (let j = 0; j < hiddenColumns.length; j++) { + const ncColumnId = ncTbl.columns.find( + (x) => x.title === hiddenColumns[j] + ).id; + const ncViewColumnId = viewDetails.find( + (x) => x.fk_column_id === ncColumnId + )?.id; + // const ncViewColumnId = await nc_getViewColumnId( + // viewId, + // viewType, + // ncColumnId + // ); + if (ncViewColumnId === undefined) continue; + + // first two positions held by record id & record hash + const _perfStart = recordPerfStart(); + await api.dbViewColumn.update(viewId, ncViewColumnId, { + show: false, + order: j + 1 + c.length, + }); + recordPerfStats(_perfStart, 'dbViewColumn.update'); + } + + // rest of the columns from airtable- retain order & visibility property + for (let j = 0; j < c.length; j++) { + const ncColumnId = await sMap.getNcIdFromAtId(c[j].columnId); + const ncViewColumnId = await nc_getViewColumnId( + viewId, + viewType, + ncColumnId + ); + if (ncViewColumnId === undefined) continue; + + // first two positions held by record id & record hash + const configData = { show: c[j].visibility, order: j + 1 }; + if (viewType === 'form') { + if (_c?.metadata?.form?.fieldsByColumnId?.[c[j].columnId]) { + const x = _c.metadata.form.fieldsByColumnId[c[j].columnId]; + const formData = { ...configData }; + if (x?.title) formData[`label`] = x.title; + if (x?.required) formData[`required`] = x.required; + if (x?.description) formData[`description`] = x.description; + const _perfStart = recordPerfStart(); + await api.dbView.formColumnUpdate(ncViewColumnId, formData); + recordPerfStats(_perfStart, 'dbView.formColumnUpdate'); + } + } + const _perfStart = recordPerfStart(); + await api.dbViewColumn.update(viewId, ncViewColumnId, configData); + recordPerfStats(_perfStart, 'dbViewColumn.update'); + } + } + + /////////////////////////////////////////////////////////////////////////////// + let recordCnt = 0; + try { + logBasic('SDK initialized'); + api = new Api({ + baseURL: syncDB.baseURL, + headers: { + 'xc-auth': syncDB.authToken, + }, + }); + + logDetailed('Project initialization started'); + // delete project if already exists + if (debugMode) await init(syncDB); + + logDetailed('Project initialized'); + + logBasic('Retrieving Airtable schema'); + // read schema file + const schema = await getAirtableSchema(syncDB); + const aTblSchema = schema.tableSchemas; + logDetailed('Project schema extraction completed'); + + if (!syncDB.projectId) { + if (!syncDB.projectName) + throw new Error('Project name or id not provided'); + // create empty project + await nocoCreateProject(syncDB.projectName); + logDetailed('Project created'); + } else { + await nocoGetProject(syncDB.projectId); + syncDB.projectName = ncCreatedProjectSchema?.title; + syncDB.baseId = syncDB.baseId || ncCreatedProjectSchema.bases[0].id; + logDetailed('Getting existing project meta'); + } + + logBasic('Importing Tables...'); + // prepare table schema (base) + await nocoCreateBaseSchema(aTblSchema); + logDetailed('Table creation completed'); + + logDetailed('Configuring Links'); + // add LTAR + await nocoCreateLinkToAnotherRecord(aTblSchema); + logDetailed('Migrating LTAR columns completed'); + + if (syncDB.options.syncLookup) { + logDetailed(`Configuring Lookup`); + // add look-ups + await nocoCreateLookups(aTblSchema); + logDetailed('Migrating Lookup columns completed'); + } + + if (syncDB.options.syncRollup) { + logDetailed('Configuring Rollup'); + // add roll-ups + await nocoCreateRollup(aTblSchema); + logDetailed('Migrating Rollup columns completed'); + + if (syncDB.options.syncLookup) { + logDetailed('Migrating Lookup form Rollup columns'); + // lookups for rollup + await nocoLookupForRollup(); + logDetailed('Migrating Lookup form Rollup columns completed'); + } + } + logDetailed('Configuring Display Value column'); + // configure Display Value + await nocoSetPrimary(aTblSchema); + logDetailed('Configuring Display Value column completed'); + + logBasic('Configuring User(s)'); + // add users + await nocoAddUsers(schema); + logDetailed('Adding users completed'); + + // hide-fields + // await nocoReconfigureFields(aTblSchema); + + logBasic('Syncing views'); + // configure views + await nocoConfigureGridView(syncDB, aTblSchema); + await nocoConfigureFormView(syncDB, aTblSchema); + await nocoConfigureGalleryView(syncDB, aTblSchema); + logDetailed('Syncing views completed'); + + if (syncDB.options.syncData) { + try { + // await nc_DumpTableSchema(); + const _perfStart = recordPerfStart(); + const ncTblList = await api.base.tableList( + ncCreatedProjectSchema.id, + syncDB.baseId + ); + recordPerfStats(_perfStart, 'base.tableList'); + + logBasic('Reading Records...'); + + const recordsMap = {}; + + for (let i = 0; i < ncTblList.list.length; i++) { + // not a migrated table, skip + if ( + undefined === + aTblSchema.find((x) => x.name === ncTblList.list[i].title) + ) + continue; + + const _perfStart = recordPerfStart(); + const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + recordPerfStats(_perfStart, 'dbTable.read'); + + recordCnt = 0; + // await nocoReadData(syncDB, ncTbl); + + recordsMap[ncTbl.id] = await importData({ + projectName: syncDB.projectName, + table: ncTbl, + base, + api, + logBasic, + nocoBaseDataProcessing_v2, + sDB: syncDB, + logDetailed, + }); + rtc.data.records += await recordsMap[ncTbl.id].getCount(); + + logDetailed(`Data inserted from ${ncTbl.title}`); + } + + logBasic('Configuring Record Links...'); + for (let i = 0; i < ncTblList.list.length; i++) { + // not a migrated table, skip + if ( + undefined === + aTblSchema.find((x) => x.name === ncTblList.list[i].title) + ) + continue; + + const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + + rtc.data.nestedLinks += await importLTARData({ + table: ncTbl, + projectName: syncDB.projectName, + api, + base, + fields: null, //Object.values(tblLinkGroup).flat(), + logBasic, + insertedAssocRef, + logDetailed, + records: recordsMap[ncTbl.id], + atNcAliasRef, + ncLinkMappingTable, + }); + } + + if (storeLinks) { + // const insertJobs: Promise[] = []; + // for (const [pTitle, v] of Object.entries(ncLinkDataStore)) { + // logBasic(`:: ${pTitle}`); + // for (const [, record] of Object.entries(v)) { + // const tbl = ncTblList.list.find(a => a.title === pTitle); + // await nocoLinkProcessing(syncDB.projectName, tbl, record, 0); + // // insertJobs.push( + // // nocoLinkProcessing(syncDB.projectName, tbl, record, 0) + // // ); + // } + // } + // await Promise.all(insertJobs); + // await nocoLinkProcessing(syncDB.projectName, 0, 0, 0); + } else { + // // create link groups (table: link fields) + // // const tblLinkGroup = {}; + // // for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { + // // const x = ncLinkMappingTable[idx]; + // // if (tblLinkGroup[x.aTbl.tblId] === undefined) + // // tblLinkGroup[x.aTbl.tblId] = [x.aTbl.name]; + // // else tblLinkGroup[x.aTbl.tblId].push(x.aTbl.name); + // // } + // // + // // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); + // // + // // await importLTARData({ + // // table: ncTbl, + // // projectName: syncDB.projectName, + // // api, + // // base, + // // fields: Object.values(tblLinkGroup).flat(), + // // logBasic + // // }); + // for (const [k, v] of Object.entries(tblLinkGroup)) { + // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); + // + // // not a migrated table, skip + // if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) + // continue; + // + // recordCnt = 0; + // await nocoReadDataSelected( + // syncDB.projectName, + // ncTbl, + // async (projName, table, record, _field) => { + // await nocoLinkProcessing(projName, table, record, _field); + // }, + // v + // ); + // } + } + } catch (error) { + logDetailed( + `There was an error while migrating data! Please make sure your API key (${syncDB.apiKey}) is correct.` + ); + logDetailed(`Error: ${error}`); + } + } + if (generate_migrationStats) { + await generateMigrationStats(aTblSchema); + } + } catch (e) { + if (e.response?.data?.msg) { + Tele.event({ + event: 'a:airtable-import:error', + data: { error: e.response.data.msg }, + }); + throw new Error(e.response.data.msg); + } + throw e; + } +}; + +export function getUniqueNameGenerator(defaultName = 'name') { + const namesRef = {}; + + return (initName: string = defaultName): string => { + let name = initName === '_' ? defaultName : initName; + let c = 0; + while (name in namesRef) { + name = `${initName}_${++c}`; + } + namesRef[name] = true; + return name; + }; +} + +export interface AirtableSyncConfig { + id: string; + baseURL: string; + authToken: string; + projectName?: string; + projectId?: string; + baseId?: string; + apiKey: string; + shareId: string; + options: { + syncViews: boolean; + syncData: boolean; + syncRollup: boolean; + syncLookup: boolean; + syncFormula: boolean; + syncAttachment: boolean; + }; +} diff --git a/packages/nocodb/src/lib/services/syncService/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/services/syncService/helpers/readAndProcessData.ts new file mode 100644 index 0000000000..ad004205d1 --- /dev/null +++ b/packages/nocodb/src/lib/services/syncService/helpers/readAndProcessData.ts @@ -0,0 +1,338 @@ +import { AirtableBase } from 'airtable/lib/airtable_base'; +import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk'; +import EntityMap from './EntityMap'; + +const BULK_DATA_BATCH_SIZE = 500; +const ASSOC_BULK_DATA_BATCH_SIZE = 1000; +const BULK_PARALLEL_PROCESS = 5; + +async function readAllData({ + table, + fields, + base, + logBasic = (_str) => {}, +}: { + table: { title?: string }; + fields?; + base: AirtableBase; + logBasic?: (string) => void; + logDetailed?: (string) => void; +}): Promise { + return new Promise((resolve, reject) => { + let data = null; + + const selectParams: any = { + pageSize: 100, + }; + + if (fields) selectParams.fields = fields; + + base(table.title) + .select(selectParams) + .eachPage( + async function page(records, fetchNextPage) { + if (!data) { + data = new EntityMap(); + await data.init(); + } + + for await (const record of records) { + await data.addRow({ id: record.id, ...record.fields }); + } + + const tmpLength = await data.getCount(); + + logBasic( + `:: Reading '${table.title}' data :: ${Math.max( + 1, + tmpLength - records.length + )} - ${tmpLength}` + ); + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + async function done(err) { + if (err) { + console.error(err); + return reject(err); + } + resolve(data); + } + ); + }); +} + +export async function importData({ + projectName, + table, + base, + api, + nocoBaseDataProcessing_v2, + sDB, + logDetailed = (_str) => {}, + logBasic = (_str) => {}, +}: { + projectName: string; + table: { title?: string; id?: string }; + fields?; + base: AirtableBase; + logBasic: (string) => void; + logDetailed: (string) => void; + api: Api; + nocoBaseDataProcessing_v2; + sDB; +}): Promise { + try { + // @ts-ignore + const records = await readAllData({ + table, + base, + logDetailed, + logBasic, + }); + + await new Promise(async (resolve) => { + const readable = records.getStream(); + const allRecordsCount = await records.getCount(); + const promises = []; + let tempData = []; + let importedCount = 0; + let activeProcess = 0; + readable.on('data', async (record) => { + promises.push( + new Promise(async (resolve) => { + activeProcess++; + if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause(); + const { id: rid, ...fields } = record; + const r = await nocoBaseDataProcessing_v2(sDB, table, { + id: rid, + fields, + }); + tempData.push(r); + + if (tempData.length >= BULK_DATA_BATCH_SIZE) { + let insertArray = tempData.splice(0, tempData.length); + await api.dbTableRow.bulkCreate( + 'nc', + projectName, + table.id, + insertArray + ); + logBasic( + `:: Importing '${ + table.title + }' data :: ${importedCount} - ${Math.min( + importedCount + BULK_DATA_BATCH_SIZE, + allRecordsCount + )}` + ); + importedCount += insertArray.length; + insertArray = []; + } + activeProcess--; + if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume(); + resolve(true); + }) + ); + }); + readable.on('end', async () => { + await Promise.all(promises); + if (tempData.length > 0) { + await api.dbTableRow.bulkCreate( + 'nc', + projectName, + table.id, + tempData + ); + logBasic( + `:: Importing '${ + table.title + }' data :: ${importedCount} - ${Math.min( + importedCount + BULK_DATA_BATCH_SIZE, + allRecordsCount + )}` + ); + importedCount += tempData.length; + tempData = []; + } + resolve(true); + }); + }); + + return records; + } catch (e) { + console.log(e); + return null; + } +} + +export async function importLTARData({ + table, + fields, + base, + api, + projectName, + insertedAssocRef = {}, + logDetailed = (_str) => {}, + logBasic = (_str) => {}, + records, + atNcAliasRef, + ncLinkMappingTable, +}: { + projectName: string; + table: { title?: string; id?: string }; + fields; + base: AirtableBase; + logDetailed: (string) => void; + logBasic: (string) => void; + api: Api; + insertedAssocRef: { [assocTableId: string]: boolean }; + records?: EntityMap; + atNcAliasRef: { + [ncTableId: string]: { + [ncTitle: string]: string; + }; + }; + ncLinkMappingTable: Record>[]; +}) { + const assocTableMetas: Array<{ + modelMeta: { id?: string; title?: string }; + colMeta: { title?: string }; + curCol: { title?: string }; + refCol: { title?: string }; + }> = []; + const allData = + records || + (await readAllData({ + table, + fields, + base, + logDetailed, + logBasic, + })); + + const modelMeta: any = await api.dbTable.read(table.id); + + for (const colMeta of modelMeta.columns) { + // skip columns which are not LTAR and Many to many + if ( + colMeta.uidt !== UITypes.LinkToAnotherRecord || + colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY + ) { + continue; + } + + // skip if already inserted + if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue; + + // self links: skip if the column under consideration is the add-on column NocoDB creates + if (ncLinkMappingTable.every((a) => a.nc.title !== colMeta.title)) continue; + + // mark as inserted + insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; + + const assocModelMeta: TableType = (await api.dbTable.read( + colMeta.colOptions.fk_mm_model_id + )) as any; + + // extract associative table and columns meta + assocTableMetas.push({ + modelMeta: assocModelMeta, + colMeta, + curCol: assocModelMeta.columns.find( + (c) => c.id === colMeta.colOptions.fk_mm_child_column_id + ), + refCol: assocModelMeta.columns.find( + (c) => c.id === colMeta.colOptions.fk_mm_parent_column_id + ), + }); + } + + let nestedLinkCnt = 0; + // Iterate over all related M2M associative table + for await (const assocMeta of assocTableMetas) { + let assocTableData = []; + let importedCount = 0; + + // extract insert data from records + await new Promise((resolve) => { + const promises = []; + const readable = allData.getStream(); + let activeProcess = 0; + readable.on('data', async (record) => { + promises.push( + new Promise(async (resolve) => { + activeProcess++; + if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause(); + const { id: _atId, ...rec } = record; + + // todo: use actual alias instead of sanitized + assocTableData.push( + ...( + rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || [] + ).map((id) => ({ + [assocMeta.curCol.title]: record.id, + [assocMeta.refCol.title]: id, + })) + ); + + if (assocTableData.length >= ASSOC_BULK_DATA_BATCH_SIZE) { + let insertArray = assocTableData.splice(0, assocTableData.length); + logBasic( + `:: Importing '${ + table.title + }' LTAR data :: ${importedCount} - ${Math.min( + importedCount + ASSOC_BULK_DATA_BATCH_SIZE, + insertArray.length + )}` + ); + + await api.dbTableRow.bulkCreate( + 'nc', + projectName, + assocMeta.modelMeta.id, + insertArray + ); + + importedCount += insertArray.length; + insertArray = []; + } + activeProcess--; + if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume(); + resolve(true); + }) + ); + }); + readable.on('end', async () => { + await Promise.all(promises); + if (assocTableData.length >= 0) { + logBasic( + `:: Importing '${ + table.title + }' LTAR data :: ${importedCount} - ${Math.min( + importedCount + ASSOC_BULK_DATA_BATCH_SIZE, + assocTableData.length + )}` + ); + + await api.dbTableRow.bulkCreate( + 'nc', + projectName, + assocMeta.modelMeta.id, + assocTableData + ); + + importedCount += assocTableData.length; + assocTableData = []; + } + resolve(true); + }); + }); + + nestedLinkCnt += importedCount; + } + return nestedLinkCnt; +} diff --git a/packages/nocodb/src/lib/services/syncService/helpers/syncMap.ts b/packages/nocodb/src/lib/services/syncService/helpers/syncMap.ts new file mode 100644 index 0000000000..7855bbaa2c --- /dev/null +++ b/packages/nocodb/src/lib/services/syncService/helpers/syncMap.ts @@ -0,0 +1,31 @@ +export const mapTbl = {}; + +// static mapping records between aTblId && ncId +export const addToMappingTbl = function addToMappingTbl( + aTblId, + ncId, + ncName, + parent? +) { + mapTbl[aTblId] = { + ncId: ncId, + ncParent: parent, + // name added to assist in quick debug + ncName: ncName, + }; +}; + +// get NcID from airtable ID +export const getNcIdFromAtId = function getNcIdFromAtId(aId) { + return mapTbl[aId]?.ncId; +}; + +// get nc Parent from airtable ID +export const getNcParentFromAtId = function getNcParentFromAtId(aId) { + return mapTbl[aId]?.ncParent; +}; + +// get nc-title from airtable ID +export const getNcNameFromAtId = function getNcNameFromAtId(aId) { + return mapTbl[aId]?.ncName; +}; diff --git a/packages/nocodb/src/lib/services/syncService/index.ts b/packages/nocodb/src/lib/services/syncService/index.ts new file mode 100644 index 0000000000..ddd27adcfd --- /dev/null +++ b/packages/nocodb/src/lib/services/syncService/index.ts @@ -0,0 +1,47 @@ +import { Tele } from 'nc-help'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse' +import { Project, SyncSource } from '../../models' + +export async function syncSourceList(param: { + projectId: string; + baseId?: string; +}) { + return new PagedResponseImpl( + await SyncSource.list(param.projectId, param.baseId) + ); +} + +export async function syncCreate(param: { + projectId: string; + baseId?: string; + userId: string; + // todo: define type + syncPayload: Partial; +}) { + Tele.emit('evt', { evt_type: 'webhooks:created' }); + const project = await Project.getWithInfo(param.projectId); + + const sync = await SyncSource.insert({ + ...param.syncPayload, + fk_user_id: param.userId, + base_id: param.baseId ? param.baseId : project.bases[0].id, + project_id: param.projectId, + }); + return sync; +} + +export async function syncDelete(param: { syncId: string }) { + Tele.emit('evt', { evt_type: 'webhooks:deleted' }); + return await SyncSource.delete(param.syncId) +} + +export async function syncUpdate(param:{ + syncId: string; + syncPayload: Partial; +}) { + Tele.emit('evt', { evt_type: 'webhooks:updated' }); + + return await SyncSource.update(param.syncId, param.syncPayload) +} + +export { default as airtableImportJob } from './helpers/job'; From 92d479386da3cdc91a5acf6389ac0b02264b9a2f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 12:40:29 +0530 Subject: [PATCH 21/64] refactor: public apis Signed-off-by: Pranav C --- .../src/lib/controllers/public/index.ts | 5 + .../lib/controllers/public/publicDataApis.ts | 104 ++++ .../public/publicDataExportApis.ts | 266 ++++++++++ .../lib/controllers/public/publicMetaApis.ts | 31 ++ packages/nocodb/src/lib/services/index.ts | 1 + .../nocodb/src/lib/services/public/index.ts | 5 + .../public/publicDataExportService.ts | 162 ++++++ .../lib/services/public/publicDataService.ts | 463 ++++++++++++++++++ .../lib/services/public/publicMetaservice.ts | 100 ++++ 9 files changed, 1137 insertions(+) create mode 100644 packages/nocodb/src/lib/controllers/public/index.ts create mode 100644 packages/nocodb/src/lib/controllers/public/publicDataApis.ts create mode 100644 packages/nocodb/src/lib/controllers/public/publicDataExportApis.ts create mode 100644 packages/nocodb/src/lib/controllers/public/publicMetaApis.ts create mode 100644 packages/nocodb/src/lib/services/public/index.ts create mode 100644 packages/nocodb/src/lib/services/public/publicDataExportService.ts create mode 100644 packages/nocodb/src/lib/services/public/publicDataService.ts create mode 100644 packages/nocodb/src/lib/services/public/publicMetaservice.ts diff --git a/packages/nocodb/src/lib/controllers/public/index.ts b/packages/nocodb/src/lib/controllers/public/index.ts new file mode 100644 index 0000000000..60028a421f --- /dev/null +++ b/packages/nocodb/src/lib/controllers/public/index.ts @@ -0,0 +1,5 @@ +import publicDataApis from './publicDataApis'; +import publicDataExportApis from './publicDataExportApis'; +import publicMetaApis from './publicMetaApis'; + +export { publicDataApis, publicDataExportApis, publicMetaApis }; diff --git a/packages/nocodb/src/lib/controllers/public/publicDataApis.ts b/packages/nocodb/src/lib/controllers/public/publicDataApis.ts new file mode 100644 index 0000000000..77ba40c157 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/public/publicDataApis.ts @@ -0,0 +1,104 @@ +import { Request, Response, Router } from 'express'; +import multer from 'multer'; +import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants'; +import catchError from '../../meta/helpers/catchError'; +import { publicDataService } from '../../services'; + +export async function dataList(req: Request, res: Response) { + const pagedResponse = publicDataService.dataList({ + query: req.query, + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + }); + res.json(pagedResponse); +} + +// todo: Handle the error case where view doesnt belong to model +async function groupedDataList(req: Request, res: Response) { + const groupedData = await publicDataService.groupedDataList({ + query: req.query, + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + groupColumnId: req.params.columnId, + }); + res.json(groupedData); +} +async function dataInsert(req: Request & { files: any[] }, res: Response) { + const insertResult = await publicDataService.dataInsert({ + sharedViewUuid: req.params.sharedViewUuid, + password: req.headers?.['xc-password'] as string, + body: req.body?.data, + siteUrl: (req as any).ncSiteUrl, + files: req.files, + }); + + res.json(insertResult); +} + +async function relDataList(req, res) { + const pagedResponse = publicDataService.relDataList({ + query: req.query, + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + columnId: req.params.columnId, + }); + + res.json(pagedResponse); +} + +export async function publicMmList(req: Request, res: Response) { + const paginatedResponse = await publicDataService.publicMmList({ + query: req.query, + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + columnId: req.params.columnId, + rowId: req.params.rowId, + }); + res.json(paginatedResponse); +} + +export async function publicHmList(req: Request, res: Response) { + const paginatedResponse = await publicDataService.publicHmList({ + query: req.query, + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + columnId: req.params.columnId, + rowId: req.params.rowId, + }); + res.json(paginatedResponse); +} + +const router = Router({ mergeParams: true }); +router.get( + '/api/v1/db/public/shared-view/:sharedViewUuid/rows', + catchError(dataList) +); +router.get( + '/api/v1/db/public/shared-view/:sharedViewUuid/group/:columnId', + catchError(groupedDataList) +); +router.get( + '/api/v1/db/public/shared-view/:sharedViewUuid/nested/:columnId', + catchError(relDataList) +); +router.post( + '/api/v1/db/public/shared-view/:sharedViewUuid/rows', + multer({ + storage: multer.diskStorage({}), + limits: { + fieldSize: NC_ATTACHMENT_FIELD_SIZE, + }, + }).any(), + catchError(dataInsert) +); + +router.get( + '/api/v1/db/public/shared-view/:sharedViewUuid/rows/:rowId/mm/:colId', + catchError(publicMmList) +); +router.get( + '/api/v1/db/public/shared-view/:sharedViewUuid/rows/:rowId/hm/:colId', + catchError(publicHmList) +); + +export default router; diff --git a/packages/nocodb/src/lib/controllers/public/publicDataExportApis.ts b/packages/nocodb/src/lib/controllers/public/publicDataExportApis.ts new file mode 100644 index 0000000000..cc677d6ab6 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/public/publicDataExportApis.ts @@ -0,0 +1,266 @@ +import { Request, Response, Router } from 'express'; +import * as XLSX from 'xlsx'; +import { nocoExecute } from 'nc-help'; +import papaparse from 'papaparse'; +import { ErrorMessages, isSystemColumn, UITypes, ViewTypes } from 'nocodb-sdk'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import catchError, { NcError } from '../../meta/helpers/catchError'; +import { + Base, + Column, + LinkToAnotherRecordColumn, + LookupColumn, + Model, + View, +} from '../../models'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; + +async function exportExcel(req: Request, res: Response) { + const view = await View.getByUUID(req.params.publicDataUuid); + if (!view) NcError.notFound('Not found'); + if ( + view.type !== ViewTypes.GRID && + view.type !== ViewTypes.KANBAN && + view.type !== ViewTypes.GALLERY && + view.type !== ViewTypes.MAP + ) + NcError.notFound('Not found'); + + if (view.password && view.password !== req.headers?.['xc-password']) { + NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const model = await view.getModelWithInfo(); + + await view.getColumns(); + + const { offset, dbRows, elapsed } = await getDbRows(model, view, req); + + const fields = req.query.fields as string[]; + + const data = XLSX.utils.json_to_sheet( + dbRows.map((o: Record) => + Object.fromEntries(fields.map((f) => [f, o[f]])) + ), + { header: fields } + ); + + const wb = XLSX.utils.book_new(); + + XLSX.utils.book_append_sheet(wb, data, view.title); + + const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); + + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + view.title + )}-export.xlsx"`, + }); + res.end(buf); +} + +async function exportCsv(req: Request, res: Response) { + const view = await View.getByUUID(req.params.publicDataUuid); + const fields = req.query.fields; + + if (!view) NcError.notFound('Not found'); + if ( + view.type !== ViewTypes.GRID && + view.type !== ViewTypes.KANBAN && + view.type !== ViewTypes.GALLERY && + view.type !== ViewTypes.MAP + ) + NcError.notFound('Not found'); + + if (view.password && view.password !== req.headers?.['xc-password']) { + NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const model = await view.getModelWithInfo(); + await view.getColumns(); + + const { offset, dbRows, elapsed } = await getDbRows(model, view, req); + + const data = papaparse.unparse( + { + fields: model.columns + .sort((c1, c2) => + Array.isArray(fields) + ? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any) + : 0 + ) + .filter( + (c) => + !fields || !Array.isArray(fields) || fields.includes(c.title as any) + ) + .map((c) => c.title), + data: dbRows, + }, + { + escapeFormulae: true, + } + ); + + res.set({ + 'Access-Control-Expose-Headers': 'nc-export-offset', + 'nc-export-offset': offset, + 'nc-export-elapsed-time': elapsed, + 'Content-Disposition': `attachment; filename="${encodeURI( + view.title + )}-export.csv"`, + }); + res.send(data); +} + +async function getDbRows(model, view: View, req: Request) { + view.model.columns = view.columns + .filter((c) => c.show) + .map( + (c) => + new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any) + ) + .filter((column) => !isSystemColumn(column) || view.show_system_fields); + + if (!model) NcError.notFound('Table not found'); + + const listArgs: any = { ...req.query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + + const base = await Base.get(model.base_id); + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const requestObj = await getAst({ + query: req.query, + model, + view, + includePkByDefault: false, + }); + + let offset = +req.query.offset || 0; + const limit = 100; + // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; + const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; + const dbRows = []; + const startTime = process.hrtime(); + let elapsed, temp; + + for ( + elapsed = 0; + elapsed < timeout; + offset += limit, + temp = process.hrtime(startTime), + elapsed = temp[0] * 1000 + temp[1] / 1000000 + ) { + const rows = await nocoExecute( + requestObj, + await baseModel.list({ ...listArgs, offset, limit }), + {}, + listArgs + ); + + if (!rows?.length) { + offset = -1; + break; + } + + for (const row of rows) { + const dbRow = { ...row }; + + for (const column of view.model.columns) { + dbRow[column.title] = await serializeCellValue({ + value: row[column.title], + column, + }); + } + dbRows.push(dbRow); + } + } + return { offset, dbRows, elapsed }; +} + +async function serializeCellValue({ + value, + column, +}: { + column?: Column; + value: any; +}) { + if (!column) { + return value; + } + + if (!value) return value; + + switch (column?.uidt) { + case UITypes.Attachment: { + let data = value; + try { + if (typeof value === 'string') { + data = JSON.parse(value); + } + } catch {} + + return (data || []).map( + (attachment) => + `${encodeURI(attachment.title)}(${encodeURI(attachment.url)})` + ); + } + case UITypes.Lookup: + { + const colOptions = await column.getColOptions(); + const lookupColumn = await colOptions.getLookupColumn(); + return ( + await Promise.all( + [...(Array.isArray(value) ? value : [value])].map(async (v) => + serializeCellValue({ + value: v, + column: lookupColumn, + }) + ) + ) + ).join(', '); + } + break; + case UITypes.LinkToAnotherRecord: + { + const colOptions = + await column.getColOptions(); + const relatedModel = await colOptions.getRelatedTable(); + await relatedModel.getColumns(); + return [...(Array.isArray(value) ? value : [value])] + .map((v) => { + return v[relatedModel.displayValue?.title]; + }) + .join(', '); + } + break; + default: + if (value && typeof value === 'object') { + return JSON.stringify(value); + } + return value; + } +} + +const router = Router({ mergeParams: true }); +router.get( + '/api/v1/db/public/shared-view/:publicDataUuid/rows/export/csv', + catchError(exportCsv) +); +router.get( + '/api/v1/db/public/shared-view/:publicDataUuid/rows/export/excel', + catchError(exportExcel) +); +export default router; diff --git a/packages/nocodb/src/lib/controllers/public/publicMetaApis.ts b/packages/nocodb/src/lib/controllers/public/publicMetaApis.ts new file mode 100644 index 0000000000..a6b261625e --- /dev/null +++ b/packages/nocodb/src/lib/controllers/public/publicMetaApis.ts @@ -0,0 +1,31 @@ +import { Request, Response, Router } from 'express'; +import catchError from '../../meta/helpers/catchError'; +import { publicMetaService } from '../../services'; + +export async function viewMetaGet(req: Request, res: Response) { + res.json( + await publicMetaService.viewMetaGet({ + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + }) + ); +} +async function publicSharedBaseGet(req, res): Promise { + res.json( + await publicMetaService.publicSharedBaseGet({ + sharedBaseUuid: req.params.sharedViewUuid, + }) + ); +} + +const router = Router({ mergeParams: true }); +router.get( + '/api/v1/db/public/shared-view/:sharedViewUuid/meta', + catchError(viewMetaGet) +); + +router.get( + '/api/v1/db/public/shared-base/:sharedBaseUuid/meta', + catchError(publicSharedBaseGet) +); +export default router; diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index 159a24f942..f5ffe2f46f 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -33,3 +33,4 @@ export * as auditService from './auditService'; export * as swaggerService from './swaggerService'; export * as userService from './userService'; export * as syncService from './syncService'; +export * from './public'; diff --git a/packages/nocodb/src/lib/services/public/index.ts b/packages/nocodb/src/lib/services/public/index.ts new file mode 100644 index 0000000000..b24c8caab5 --- /dev/null +++ b/packages/nocodb/src/lib/services/public/index.ts @@ -0,0 +1,5 @@ +import * as publicDataService from './publicDataService'; +import * as publicDataExportApis from './publicDataExportService'; +import * as publicMetaService from './publicMetaservice'; + +export { publicDataService, publicDataExportApis, publicMetaService }; diff --git a/packages/nocodb/src/lib/services/public/publicDataExportService.ts b/packages/nocodb/src/lib/services/public/publicDataExportService.ts new file mode 100644 index 0000000000..900dd3ed88 --- /dev/null +++ b/packages/nocodb/src/lib/services/public/publicDataExportService.ts @@ -0,0 +1,162 @@ +import { nocoExecute } from 'nc-help'; +import { isSystemColumn, UITypes } from 'nocodb-sdk'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import { NcError } from '../../meta/helpers/catchError'; +import { + Base, + Column, + LinkToAnotherRecordColumn, + LookupColumn, + Model, + View, +} from '../../models'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; + +export async function getDbRows(param: { + model; + view: View; + query: any; + offset?: number; +}) { + param.view.model.columns = param.view.columns + .filter((c) => c.show) + .map( + (c) => + new Column({ + ...c, + ...param.view.model.columnsById[c.fk_column_id], + } as any) + ) + .filter( + (column) => !isSystemColumn(column) || param.view.show_system_fields + ); + + if (!param.model) NcError.notFound('Table not found'); + + const listArgs: any = { ...param.query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + + const base = await Base.get(param.model.base_id); + const baseModel = await Model.getBaseModelSQL({ + id: param.model.id, + viewId: param.view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const requestObj = await getAst({ + query: param.query, + model: param.model, + view: param.view, + includePkByDefault: false, + }); + + let offset = +param.offset || 0; + const limit = 100; + // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; + const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; + const dbRows = []; + const startTime = process.hrtime(); + let elapsed, temp; + + for ( + elapsed = 0; + elapsed < timeout; + offset += limit, + temp = process.hrtime(startTime), + elapsed = temp[0] * 1000 + temp[1] / 1000000 + ) { + const rows = await nocoExecute( + requestObj, + await baseModel.list({ ...listArgs, offset, limit }), + {}, + listArgs + ); + + if (!rows?.length) { + offset = -1; + break; + } + + for (const row of rows) { + const dbRow = { ...row }; + + for (const column of param.view.model.columns) { + dbRow[column.title] = await serializeCellValue({ + value: row[column.title], + column, + }); + } + dbRows.push(dbRow); + } + } + return { offset, dbRows, elapsed }; +} + +export async function serializeCellValue({ + value, + column, +}: { + column?: Column; + value: any; +}) { + if (!column) { + return value; + } + + if (!value) return value; + + switch (column?.uidt) { + case UITypes.Attachment: { + let data = value; + try { + if (typeof value === 'string') { + data = JSON.parse(value); + } + } catch {} + + return (data || []).map( + (attachment) => + `${encodeURI(attachment.title)}(${encodeURI(attachment.url)})` + ); + } + case UITypes.Lookup: + { + const colOptions = await column.getColOptions(); + const lookupColumn = await colOptions.getLookupColumn(); + return ( + await Promise.all( + [...(Array.isArray(value) ? value : [value])].map(async (v) => + serializeCellValue({ + value: v, + column: lookupColumn, + }) + ) + ) + ).join(', '); + } + break; + case UITypes.LinkToAnotherRecord: + { + const colOptions = + await column.getColOptions(); + const relatedModel = await colOptions.getRelatedTable(); + await relatedModel.getColumns(); + return [...(Array.isArray(value) ? value : [value])] + .map((v) => { + return v[relatedModel.displayValue?.title]; + }) + .join(', '); + } + break; + default: + if (value && typeof value === 'object') { + return JSON.stringify(value); + } + return value; + } +} diff --git a/packages/nocodb/src/lib/services/public/publicDataService.ts b/packages/nocodb/src/lib/services/public/publicDataService.ts new file mode 100644 index 0000000000..238f5e5825 --- /dev/null +++ b/packages/nocodb/src/lib/services/public/publicDataService.ts @@ -0,0 +1,463 @@ +import { nocoExecute } from 'nc-help'; +import { ErrorMessages, UITypes, ViewTypes } from 'nocodb-sdk'; +import path from 'path'; +import { nanoid } from 'nanoid'; +import slash from 'slash'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import { NcError } from '../../meta/helpers/catchError'; +import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import { + Base, + Column, + LinkToAnotherRecordColumn, + Model, + View, +} from '../../models'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { mimeIcons } from '../../utils/mimeTypes'; +import { sanitizeUrlPath } from '../attachmentService'; +import { getColumnByIdOrName } from '../dataService/helpers'; + +export async function dataList(param: { + sharedViewUuid: string; + password?: string; + query: any; +}) { + const view = await View.getByUUID(param.sharedViewUuid); + + if (!view) NcError.notFound('Not found'); + if ( + view.type !== ViewTypes.GRID && + view.type !== ViewTypes.KANBAN && + view.type !== ViewTypes.GALLERY && + view.type !== ViewTypes.MAP + ) { + NcError.notFound('Not found'); + } + + if (view.password && view.password !== param.password) { + return NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id, + }); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const listArgs: any = { ...param.query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + + let data = []; + let count = 0; + + try { + data = await nocoExecute( + await getAst({ + query: param.query, + model, + view, + }), + await baseModel.list(listArgs), + {}, + listArgs + ); + count = await baseModel.count(listArgs); + } catch (e) { + // show empty result instead of throwing error here + // e.g. search some text in a numeric field + } + + return new PagedResponseImpl(data, { ...param.query, count }); +} + +// todo: Handle the error case where view doesnt belong to model +export async function groupedDataList(param: { + sharedViewUuid: string; + password?: string; + query: any; + groupColumnId: string; +}) { + const view = await View.getByUUID(param.sharedViewUuid); + + if (!view) NcError.notFound('Not found'); + + if ( + view.type !== ViewTypes.GRID && + view.type !== ViewTypes.KANBAN && + view.type !== ViewTypes.GALLERY + ) { + NcError.notFound('Not found'); + } + + if (view.password && view.password !== param.password) { + return NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id, + }); + + return await getGroupedDataList({ + model, + view, + query: param.query, + groupColumnId: param.groupColumnId, + }); +} + +async function getGroupedDataList(param: { + model: Model; + view: View; + query: any; + groupColumnId: string; +}) { + const { model, view, query = {}, groupColumnId } = param; + const base = await Base.get(param.model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const requestObj = await getAst({ model, query: param.query, view }); + + const listArgs: any = { ...query }; + try { + listArgs.filterArr = JSON.parse(listArgs.filterArrJson); + } catch (e) {} + try { + listArgs.sortArr = JSON.parse(listArgs.sortArrJson); + } catch (e) {} + try { + listArgs.options = JSON.parse(listArgs.optionsArrJson); + } catch (e) {} + + let data = []; + + try { + const groupedData = await baseModel.groupedList({ + ...listArgs, + groupColumnId, + }); + data = await nocoExecute( + { key: 1, value: requestObj }, + groupedData, + {}, + listArgs + ); + const countArr = await baseModel.groupedListCount({ + ...listArgs, + groupColumnId, + }); + data = data.map((item) => { + // todo: use map to avoid loop + const count = + countArr.find((countItem: any) => countItem.key === item.key)?.count ?? + 0; + + item.value = new PagedResponseImpl(item.value, { + ...query, + count: count, + }); + return item; + }); + } catch (e) { + console.log(e); + NcError.internalServerError('Internal Server Error'); + } + return data; +} + +export async function dataInsert(param: { + sharedViewUuid: string; + password?: string; + body: any; + files: any[]; + siteUrl: string; +}) { + const view = await View.getByUUID(param.sharedViewUuid); + + if (!view) NcError.notFound(); + if (view.type !== ViewTypes.FORM) NcError.notFound(); + + if (view.password && view.password !== param.password) { + return NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const model = await Model.getByIdOrName({ + id: view?.fk_model_id, + }); + + const base = await Base.get(model.base_id); + const project = await base.getProject(); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + await view.getViewWithInfo(); + await view.getColumns(); + await view.getModelWithInfo(); + await view.model.getColumns(); + + const fields = (view.model.columns = view.columns + .filter((c) => c.show) + .reduce((o, c) => { + o[view.model.columnsById[c.fk_column_id].title] = new Column({ + ...c, + ...view.model.columnsById[c.fk_column_id], + } as any); + return o; + }, {}) as any); + + let body = param?.body; + + if (typeof body === 'string') body = JSON.parse(body); + + const insertObject = Object.entries(body).reduce((obj, [key, val]) => { + if (key in fields) { + obj[key] = val; + } + return obj; + }, {}); + + const attachments = {}; + const storageAdapter = await NcPluginMgrv2.storageAdapter(); + + for (const file of param.files || []) { + // remove `_` prefix and `[]` suffix + const fieldName = file?.fieldname?.replace(/^_|\[\d*]$/g, ''); + + const filePath = sanitizeUrlPath([ + 'v1', + project.title, + model.title, + fieldName, + ]); + + if (fieldName in fields && fields[fieldName].uidt === UITypes.Attachment) { + attachments[fieldName] = attachments[fieldName] || []; + const fileName = `${nanoid(6)}_${file.originalname}`; + let url = await storageAdapter.fileCreate( + slash(path.join('nc', 'uploads', ...filePath, fileName)), + file + ); + + if (!url) { + url = `${param.siteUrl}/download/${filePath.join('/')}/${fileName}`; + } + + attachments[fieldName].push({ + url, + title: file.originalname, + mimetype: file.mimetype, + size: file.size, + icon: mimeIcons[path.extname(file.originalname).slice(1)] || undefined, + }); + } + } + + for (const [column, data] of Object.entries(attachments)) { + insertObject[column] = JSON.stringify(data); + } + + return await baseModel.nestedInsert(insertObject, null); +} + +export async function relDataList(param: { + query: any; + sharedViewUuid: string; + password?: string; + columnId: string; +}) { + const view = await View.getByUUID(param.sharedViewUuid); + + if (!view) NcError.notFound('Not found'); + + if (view.type !== ViewTypes.FORM) NcError.notFound('Not found'); + + if (view.password && view.password !== param.password) { + NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const column = await Column.get({ colId: param.columnId }); + const colOptions = await column.getColOptions(); + + const model = await colOptions.getRelatedTable(); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const requestObj = await getAst({ + query: param.query, + model, + extractOnlyPrimaries: true, + }); + + let data = []; + let count = 0; + try { + data = data = await nocoExecute( + requestObj, + await baseModel.list(param.query), + {}, + param.query + ); + count = await baseModel.count(param.query); + } catch (e) { + // show empty result instead of throwing error here + // e.g. search some text in a numeric field + } + + return new PagedResponseImpl(data, { ...param.query, count }); +} + +export async function publicMmList(param: { + query: any; + sharedViewUuid: string; + password?: string; + columnId: string; + rowId: string; +}) { + const view = await View.getByUUID(param.sharedViewUuid); + + if (!view) NcError.notFound('Not found'); + if (view.type !== ViewTypes.GRID) NcError.notFound('Not found'); + + if (view.password && view.password !== param.password) { + NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const column = await getColumnByIdOrName( + param.columnId, + await view.getModel() + ); + + if (column.fk_model_id !== view.fk_model_id) + NcError.badRequest("Column doesn't belongs to the model"); + + const base = await Base.get(view.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: view.fk_model_id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const key = `List`; + const requestObj: any = { + [key]: 1, + }; + + const data = ( + await nocoExecute( + requestObj, + { + [key]: async (args) => { + return await baseModel.mmList( + { + colId: param.columnId, + parentId: param.rowId, + }, + args + ); + }, + }, + {}, + + { nested: { [key]: param.query } } + ) + )?.[key]; + + const count: any = await baseModel.mmListCount({ + colId: param.columnId, + parentId: param.rowId, + }); + + return new PagedResponseImpl(data, { ...param.query, count }); +} + +export async function publicHmList(param: { + query: any; + rowId: string; + sharedViewUuid: string; + password?: string; + columnId: string; +}) { + const view = await View.getByUUID(param.sharedViewUuid); + + if (!view) NcError.notFound('Not found'); + if (view.type !== ViewTypes.GRID) NcError.notFound('Not found'); + + if (view.password && view.password !== param.password) { + NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + const column = await getColumnByIdOrName( + param.columnId, + await view.getModel() + ); + + if (column.fk_model_id !== view.fk_model_id) + NcError.badRequest("Column doesn't belongs to the model"); + + const base = await Base.get(view.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: view.fk_model_id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const key = `List`; + const requestObj: any = { + [key]: 1, + }; + + const data = ( + await nocoExecute( + requestObj, + { + [key]: async (args) => { + return await baseModel.hmList( + { + colId: param.columnId, + id: param.rowId, + }, + args + ); + }, + }, + {}, + { nested: { [key]: param.query } } + ) + )?.[key]; + + const count = await baseModel.hmListCount({ + colId: param.columnId, + id: param.rowId, + }); + + return new PagedResponseImpl(data, { ...param.query, count }); +} diff --git a/packages/nocodb/src/lib/services/public/publicMetaservice.ts b/packages/nocodb/src/lib/services/public/publicMetaservice.ts new file mode 100644 index 0000000000..940b03e877 --- /dev/null +++ b/packages/nocodb/src/lib/services/public/publicMetaservice.ts @@ -0,0 +1,100 @@ +import { + ErrorMessages, + LinkToAnotherRecordType, + RelationTypes, + UITypes, +} from 'nocodb-sdk'; +import { NcError } from '../../meta/helpers/catchError'; +import { + Base, + Column, + LinkToAnotherRecordColumn, + Model, + Project, + View, +} from '../../models'; + +export async function viewMetaGet(param: { + sharedViewUuid: string; + password: string; +}) { + const view: View & { + relatedMetas?: { [ket: string]: Model }; + client?: string; + } = await View.getByUUID(param.sharedViewUuid); + + if (!view) NcError.notFound('Not found'); + + if (view.password && view.password !== param.password) { + NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); + } + + await view.getFilters(); + await view.getSorts(); + + await view.getViewWithInfo(); + await view.getColumns(); + await view.getModelWithInfo(); + await view.model.getColumns(); + + const base = await Base.get(view.model.base_id); + view.client = base.type; + + // todo: return only required props + delete view['password']; + + view.model.columns = view.columns + .filter((c) => { + const column = view.model.columnsById[c.fk_column_id]; + return ( + c.show || + (column.rqd && !column.cdf && !column.ai) || + column.pk || + view.model.columns.some( + (c1) => + c1.uidt === UITypes.LinkToAnotherRecord && + (c1.colOptions).type === + RelationTypes.BELONGS_TO && + view.columns.some((vc) => vc.fk_column_id === c1.id && vc.show) && + (c1.colOptions).fk_child_column_id === + c.fk_column_id + ) + ); + }) + .map( + (c) => + new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any) + ) as any; + + const relatedMetas = {}; + + // load related table metas + for (const col of view.model.columns) { + if (UITypes.LinkToAnotherRecord === col.uidt) { + const colOpt = await col.getColOptions(); + relatedMetas[colOpt.fk_related_model_id] = await Model.getWithInfo({ + id: colOpt.fk_related_model_id, + }); + if (colOpt.type === 'mm') { + relatedMetas[colOpt.fk_mm_model_id] = await Model.getWithInfo({ + id: colOpt.fk_mm_model_id, + }); + } + } + } + + view.relatedMetas = relatedMetas; + + return view; +} +export async function publicSharedBaseGet(param: { + sharedBaseUuid: string; +}): Promise { + const project = await Project.getByUuid(param.sharedBaseUuid); + + if (!project) { + NcError.notFound(); + } + + return { project_id: project.id }; +} From 2e360166b3a40b6e92ce11387d1d355687a230c8 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 13:03:20 +0530 Subject: [PATCH 22/64] refactor: map new controller Signed-off-by: Pranav C --- .../src/lib/controllers/public/index.ts | 8 +- .../lib/controllers/userController/index.ts | 2 +- .../src/lib/controllers/utilController.ts | 363 +----------------- .../nocodb/src/lib/controllers/utilService.ts | 59 --- 4 files changed, 24 insertions(+), 408 deletions(-) delete mode 100644 packages/nocodb/src/lib/controllers/utilService.ts diff --git a/packages/nocodb/src/lib/controllers/public/index.ts b/packages/nocodb/src/lib/controllers/public/index.ts index 60028a421f..60e7849dc2 100644 --- a/packages/nocodb/src/lib/controllers/public/index.ts +++ b/packages/nocodb/src/lib/controllers/public/index.ts @@ -1,5 +1,5 @@ -import publicDataApis from './publicDataApis'; -import publicDataExportApis from './publicDataExportApis'; -import publicMetaApis from './publicMetaApis'; +import publicDataController from './publicDataApis'; +import publicDataExportController from './publicDataExportApis'; +import publicMetaController from './publicMetaApis'; -export { publicDataApis, publicDataExportApis, publicMetaApis }; +export { publicDataController, publicDataExportController, publicMetaController }; diff --git a/packages/nocodb/src/lib/controllers/userController/index.ts b/packages/nocodb/src/lib/controllers/userController/index.ts index 29a53d781e..0b42c7c9c3 100644 --- a/packages/nocodb/src/lib/controllers/userController/index.ts +++ b/packages/nocodb/src/lib/controllers/userController/index.ts @@ -458,4 +458,4 @@ const mapRoutes = (router) => { // respond with password reset page router.get('/auth/password/reset/:tokenId', catchError(renderPasswordReset)); }; -export { mapRoutes as userApis }; +export { mapRoutes as userController }; diff --git a/packages/nocodb/src/lib/controllers/utilController.ts b/packages/nocodb/src/lib/controllers/utilController.ts index 93ad927e1c..e5e2b504b6 100644 --- a/packages/nocodb/src/lib/controllers/utilController.ts +++ b/packages/nocodb/src/lib/controllers/utilController.ts @@ -1,372 +1,47 @@ // // Project CRUD import { Request, Response } from 'express'; -import { compareVersions, validate } from 'compare-versions'; - -import { ViewTypes } from 'nocodb-sdk'; -import Project from '../models/Project'; -import Noco from '../Noco'; -import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; -import { MetaTable } from '../utils/globals'; -import { packageVersion } from '../utils/packageVersion'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2'; -import NcConfigFactory, { - defaultConnectionConfig, -} from '../utils/NcConfigFactory'; -import User from '../models/User'; import catchError from '../meta/helpers/catchError'; -import axios from 'axios'; -import { NC_ATTACHMENT_FIELD_SIZE } from '../constants'; - -const versionCache = { - releaseVersion: null, - lastFetched: null, -}; +import { utilService } from '../services'; export async function testConnection(req: Request, res: Response) { - res.json(await SqlMgrv2.testConnection(req.body)); + res.json(await utilService.testConnection({ body: req.body })); } export async function appInfo(req: Request, res: Response) { - const projectHasAdmin = !(await User.isFirst()); - const result = { - authType: 'jwt', - projectHasAdmin, - firstUser: !projectHasAdmin, - type: 'rest', - env: process.env.NODE_ENV, - googleAuthEnabled: !!( - process.env.NC_GOOGLE_CLIENT_ID && process.env.NC_GOOGLE_CLIENT_SECRET - ), - githubAuthEnabled: !!( - process.env.NC_GITHUB_CLIENT_ID && process.env.NC_GITHUB_CLIENT_SECRET - ), - oneClick: !!process.env.NC_ONE_CLICK, - connectToExternalDB: !process.env.NC_CONNECT_TO_EXTERNAL_DB_DISABLED, - version: packageVersion, - defaultLimit: Math.max( - Math.min( - +process.env.DB_QUERY_LIMIT_DEFAULT || 25, - +process.env.DB_QUERY_LIMIT_MAX || 100 - ), - +process.env.DB_QUERY_LIMIT_MIN || 1 - ), - timezone: defaultConnectionConfig.timezone, - ncMin: !!process.env.NC_MIN, - teleEnabled: process.env.NC_DISABLE_TELE === 'true' ? false : true, - auditEnabled: process.env.NC_DISABLE_AUDIT === 'true' ? false : true, - ncSiteUrl: (req as any).ncSiteUrl, - ee: Noco.isEE(), - ncAttachmentFieldSize: NC_ATTACHMENT_FIELD_SIZE, - ncMaxAttachmentsAllowed: +(process.env.NC_MAX_ATTACHMENTS_ALLOWED || 10), - }; - - res.json(result); + res.json( + await utilService.appInfo({ + req: { + ncSiteUrl: (req as any).ncSiteUrl, + }, + }) + ); } export async function versionInfo(_req: Request, res: Response) { - if ( - !versionCache.lastFetched || - (versionCache.lastFetched && - versionCache.lastFetched < Date.now() - 1000 * 60 * 60) - ) { - const nonBetaTags = await axios - .get('https://api.github.com/repos/nocodb/nocodb/tags', { - timeout: 5000, - }) - .then((response) => { - return response.data - .map((x) => x.name) - .filter( - (v) => validate(v) && !v.includes('finn') && !v.includes('beta') - ) - .sort((x, y) => compareVersions(y, x)); - }) - .catch(() => null); - if (nonBetaTags && nonBetaTags.length > 0) { - versionCache.releaseVersion = nonBetaTags[0]; - } - versionCache.lastFetched = Date.now(); - } - - const response = { - currentVersion: packageVersion, - releaseVersion: versionCache.releaseVersion, - }; - - res.json(response); + res.json(await utilService.versionInfo()); } export async function appHealth(_: Request, res: Response) { - res.json({ - message: 'OK', - timestamp: Date.now(), - uptime: process.uptime(), - }); -} - -async function _axiosRequestMake(req: Request, res: Response) { - const { apiMeta } = req.body; - - if (apiMeta?.body) { - try { - apiMeta.body = JSON.parse(apiMeta.body); - } catch (e) { - console.log(e); - } - } - - if (apiMeta?.auth) { - try { - apiMeta.auth = JSON.parse(apiMeta.auth); - } catch (e) { - console.log(e); - } - } - - apiMeta.response = {}; - const _req = { - params: apiMeta.parameters - ? apiMeta.parameters.reduce((paramsObj, param) => { - if (param.name && param.enabled) { - paramsObj[param.name] = param.value; - } - return paramsObj; - }, {}) - : {}, - url: apiMeta.url, - method: apiMeta.method || 'GET', - data: apiMeta.body || {}, - headers: apiMeta.headers - ? apiMeta.headers.reduce((headersObj, header) => { - if (header.name && header.enabled) { - headersObj[header.name] = header.value; - } - return headersObj; - }, {}) - : {}, - responseType: apiMeta.responseType || 'json', - withCredentials: true, - }; - const data = await require('axios')(_req); - return res.json(data?.data); + res.json(await utilService.appHealth()); } export async function axiosRequestMake(req: Request, res: Response) { - const { - apiMeta: { url }, - } = req.body; - const isExcelImport = /.*\.(xls|xlsx|xlsm|ods|ots)/; - const isCSVImport = /.*\.(csv)/; - const ipBlockList = - /(10)(\.([2]([0-5][0-5]|[01234][6-9])|[1][0-9][0-9]|[1-9][0-9]|[0-9])){3}|(172)\.(1[6-9]|2[0-9]|3[0-1])(\.(2[0-4][0-9]|25[0-5]|[1][0-9][0-9]|[1-9][0-9]|[0-9])){2}|(192)\.(168)(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])){2}|(0.0.0.0)|localhost?/g; - if ( - ipBlockList.test(url) || - (!isCSVImport.test(url) && !isExcelImport.test(url)) - ) { - return res.json({}); - } - if (isCSVImport || isExcelImport) { - req.body.apiMeta.responseType = 'arraybuffer'; - } - return await _axiosRequestMake(req, res); -} - -export async function urlToDbConfig(req: Request, res: Response) { - const { url } = req.body; - try { - const connectionConfig = NcConfigFactory.extractXcUrlFromJdbc(url, true); - return res.json(connectionConfig); - } catch (error) { - return res.sendStatus(500); - } -} - -interface ViewCount { - formCount: number | null; - gridCount: number | null; - galleryCount: number | null; - kanbanCount: number | null; - total: number | null; - sharedFormCount: number | null; - sharedGridCount: number | null; - sharedGalleryCount: number | null; - sharedKanbanCount: number | null; - sharedTotal: number | null; - sharedLockedCount: number | null; -} - -interface AllMeta { - projectCount: number; - projects: ( - | { - external?: boolean | null; - tableCount: { - table: number; - view: number; - } | null; - viewCount: ViewCount; - webhookCount: number | null; - filterCount: number | null; - sortCount: number | null; - rowCount: ({ totalRecords: number } | null)[] | null; - userCount: number | null; - } - | { error: string } - )[]; - userCount: number; - sharedBaseCount: number; + res.json(await utilService.axiosRequestMake({ body: req.body })); } export async function aggregatedMetaInfo(_req: Request, res: Response) { - const [projects, userCount] = await Promise.all([ - Project.list({}), - Noco.ncMeta.metaCount(null, null, MetaTable.USERS), - ]); - - const result: AllMeta = { - projectCount: projects.length, - projects: [], - userCount, - sharedBaseCount: 0, - }; - - result.projects.push( - ...extractResultOrNull( - await Promise.allSettled( - projects.map(async (project) => { - if (project.uuid) result.sharedBaseCount++; - const [ - tableCount, - dbViewCount, - viewCount, - webhookCount, - filterCount, - sortCount, - rowCount, - userCount, - ] = extractResultOrNull( - await Promise.allSettled([ - // db tables count - Noco.ncMeta.metaCount(project.id, null, MetaTable.MODELS, { - condition: { - type: 'table', - }, - }), - // db views count - Noco.ncMeta.metaCount(project.id, null, MetaTable.MODELS, { - condition: { - type: 'view', - }, - }), - // views count - (async () => { - const views = await Noco.ncMeta.metaList2( - project.id, - null, - MetaTable.VIEWS - ); - // grid, form, gallery, kanban and shared count - return views.reduce( - (out, view) => { - out.total++; - - switch (view.type) { - case ViewTypes.GRID: - out.gridCount++; - if (view.uuid) out.sharedGridCount++; - break; - case ViewTypes.FORM: - out.formCount++; - if (view.uuid) out.sharedFormCount++; - break; - case ViewTypes.GALLERY: - out.galleryCount++; - if (view.uuid) out.sharedGalleryCount++; - break; - case ViewTypes.KANBAN: - out.kanbanCount++; - if (view.uuid) out.sharedKanbanCount++; - } - - if (view.uuid) { - if (view.password) out.sharedLockedCount++; - out.sharedTotal++; - } - - return out; - }, - { - formCount: 0, - gridCount: 0, - galleryCount: 0, - kanbanCount: 0, - total: 0, - sharedFormCount: 0, - sharedGridCount: 0, - sharedGalleryCount: 0, - sharedKanbanCount: 0, - sharedTotal: 0, - sharedLockedCount: 0, - } - ); - })(), - // webhooks count - Noco.ncMeta.metaCount(project.id, null, MetaTable.HOOKS), - // filters count - Noco.ncMeta.metaCount(project.id, null, MetaTable.FILTER_EXP), - // sorts count - Noco.ncMeta.metaCount(project.id, null, MetaTable.SORT), - // row count per base - project.getBases().then(async (bases) => { - return extractResultOrNull( - await Promise.allSettled( - bases.map(async (base) => - (await NcConnectionMgrv2.getSqlClient(base)) - .totalRecords?.() - ?.then((result) => result?.data) - ) - ) - ); - }), - // project users count - Noco.ncMeta.metaCount(null, null, MetaTable.PROJECT_USERS, { - condition: { - project_id: project.id, - }, - aggField: '*', - }), - ]) - ); + res.json(await utilService.aggregatedMetaInfo()); +} - return { - tableCount: { table: tableCount, view: dbViewCount }, - external: !project.is_meta, - viewCount, - webhookCount, - filterCount, - sortCount, - rowCount, - userCount, - }; - }) - ) - ) +export async function urlToDbConfig(req: Request, res: Response) { + res.json( + await utilService.urlToDbConfig({ + body: req.body, + }) ); - - res.json(result); } -const extractResultOrNull = (results: PromiseSettledResult[]) => { - return results.map((result) => { - if (result.status === 'fulfilled') { - return result.value; - } - console.log(result.reason); - return null; - }); -}; - export default (router) => { router.post( '/api/v1/db/meta/connection/test', diff --git a/packages/nocodb/src/lib/controllers/utilService.ts b/packages/nocodb/src/lib/controllers/utilService.ts deleted file mode 100644 index e5e2b504b6..0000000000 --- a/packages/nocodb/src/lib/controllers/utilService.ts +++ /dev/null @@ -1,59 +0,0 @@ -// // Project CRUD -import { Request, Response } from 'express'; -import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import catchError from '../meta/helpers/catchError'; -import { utilService } from '../services'; - -export async function testConnection(req: Request, res: Response) { - res.json(await utilService.testConnection({ body: req.body })); -} - -export async function appInfo(req: Request, res: Response) { - res.json( - await utilService.appInfo({ - req: { - ncSiteUrl: (req as any).ncSiteUrl, - }, - }) - ); -} - -export async function versionInfo(_req: Request, res: Response) { - res.json(await utilService.versionInfo()); -} - -export async function appHealth(_: Request, res: Response) { - res.json(await utilService.appHealth()); -} - -export async function axiosRequestMake(req: Request, res: Response) { - res.json(await utilService.axiosRequestMake({ body: req.body })); -} - -export async function aggregatedMetaInfo(_req: Request, res: Response) { - res.json(await utilService.aggregatedMetaInfo()); -} - -export async function urlToDbConfig(req: Request, res: Response) { - res.json( - await utilService.urlToDbConfig({ - body: req.body, - }) - ); -} - -export default (router) => { - router.post( - '/api/v1/db/meta/connection/test', - ncMetaAclMw(testConnection, 'testConnection') - ); - router.get('/api/v1/db/meta/nocodb/info', catchError(appInfo)); - router.post('/api/v1/db/meta/axiosRequestMake', catchError(axiosRequestMake)); - router.get('/api/v1/version', catchError(versionInfo)); - router.get('/api/v1/health', catchError(appHealth)); - router.post('/api/v1/url_to_config', catchError(urlToDbConfig)); - router.get( - '/api/v1/aggregated-meta-info', - ncMetaAclMw(aggregatedMetaInfo, 'aggregatedMetaInfo') - ); -}; From f7fdedbca6f747e872d3a130f0c707745e4b6aa4 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 13:56:50 +0530 Subject: [PATCH 23/64] fix: type corrections Signed-off-by: Pranav C --- packages/nocodb-sdk/src/lib/Api.ts | 52 +- packages/nocodb/package-lock.json | 14 +- packages/nocodb/package.json | 2 +- packages/nocodb/src/lib/Noco.ts | 8 +- .../src/lib/controllers/columnController.ts | 25 +- .../dataApis/dataAliasExportApis.ts | 2 +- .../src/lib/controllers/dataApis/helpers.ts | 161 ++--- .../lib/controllers/dataController/export.ts | 10 +- .../lib/controllers/hookFilterController.ts | 130 ++-- .../lib/controllers/kanbanViewController.ts | 6 +- .../lib/controllers/orgLicenseController.ts | 2 +- .../src/lib/controllers/pluginController.ts | 11 +- .../src/lib/controllers/projectController.ts | 10 +- .../src/lib/controllers/public/index.ts | 6 +- .../src/lib/controllers/sortController.ts | 2 +- .../controllers/swaggerController/index.ts | 30 +- .../src/lib/controllers/sync/helpers/job.ts | 6 +- .../lib/controllers/sync/syncSourceApis.ts | 16 +- .../src/lib/controllers/tableController.ts | 2 +- .../controllers/userApi/initAdminFromEnv.ts | 16 +- .../src/lib/controllers/userApi/userApis.ts | 30 +- .../lib/controllers/userController/index.ts | 4 +- .../src/lib/controllers/viewController.ts | 5 +- .../src/lib/db/sql-client/lib/KnexClient.ts | 4 +- packages/nocodb/src/lib/db/sql-mgr/SqlMgr.ts | 4 +- packages/nocodb/src/lib/meta/NcMetaMgr.ts | 48 +- packages/nocodb/src/lib/meta/NcMetaMgrEE.ts | 8 +- .../nocodb/src/lib/meta/api/projectApis.ts | 0 .../nocodb/src/lib/meta/helpers/apiMetrics.ts | 4 +- packages/nocodb/src/lib/models/GalleryView.ts | 2 +- packages/nocodb/src/lib/models/Hook.ts | 166 +++-- packages/nocodb/src/lib/models/KanbanView.ts | 8 +- packages/nocodb/src/lib/models/Model.ts | 577 +++++++++--------- packages/nocodb/src/lib/models/Project.ts | 2 +- .../src/lib/services/apiTokenService.ts | 6 +- .../src/lib/services/attachmentService.ts | 6 +- .../nocodb/src/lib/services/baseService.ts | 10 +- .../nocodb/src/lib/services/cacheService.ts | 6 +- .../nocodb/src/lib/services/columnService.ts | 50 +- .../src/lib/services/dataService/export.ts | 30 +- .../src/lib/services/dataService/helpers.ts | 172 ++---- .../src/lib/services/dataService/index.ts | 5 +- .../nocodb/src/lib/services/filterService.ts | 57 +- .../src/lib/services/formViewColumnService.ts | 4 +- .../src/lib/services/formViewService.ts | 6 +- .../src/lib/services/galleryViewService.ts | 8 +- .../src/lib/services/gridViewColumnService.ts | 4 +- .../src/lib/services/gridViewService.ts | 6 +- .../src/lib/services/hookFilterService.ts | 8 +- .../nocodb/src/lib/services/hookService.ts | 16 +- .../src/lib/services/kanbanViewService.ts | 6 +- .../nocodb/src/lib/services/mapViewService.ts | 25 +- .../src/lib/services/metaDiffService.ts | 10 +- .../lib/services/modelVisibilityService.ts | 6 +- .../src/lib/services/orgLicenseService.ts | 8 +- .../src/lib/services/orgTokenService.ts | 6 +- .../nocodb/src/lib/services/orgUserService.ts | 4 +- .../nocodb/src/lib/services/pluginService.ts | 29 +- .../nocodb/src/lib/services/projectService.ts | 14 +- .../src/lib/services/projectUserService.ts | 4 +- .../src/lib/services/sharedBaseService.ts | 17 +- .../nocodb/src/lib/services/sortService.ts | 9 +- .../services/swaggerService/getSwaggerJSON.ts | 2 +- .../src/lib/services/swaggerService/index.ts | 13 +- .../lib/services/syncService/helpers/job.ts | 6 +- .../src/lib/services/syncService/index.ts | 18 +- .../nocodb/src/lib/services/tableService.ts | 9 +- .../src/lib/services/userService/helpers.ts | 4 +- .../src/lib/services/userService/index.ts | 87 ++- .../services/userService/initAdminFromEnv.ts | 15 +- .../userService/ui/auth/emailVerify.ts | 70 +++ .../userService/ui/auth/resetPassword.ts | 108 ++++ .../ui/emailTemplates/forgotPassword.ts | 171 ++++++ .../userService/ui/emailTemplates/invite.ts | 208 +++++++ .../userService/ui/emailTemplates/verify.ts | 207 +++++++ .../nocodb/src/lib/services/utilService.ts | 4 +- .../src/lib/services/viewColumnService.ts | 8 +- .../nocodb/src/lib/services/viewService.ts | 32 +- .../src/lib/utils/common/BaseApiBuilder.ts | 6 +- .../src/lib/v1-legacy/NcProjectBuilder.ts | 4 +- .../src/lib/v1-legacy/rest/RestAuthCtrl.ts | 18 +- .../src/lib/v1-legacy/rest/RestAuthCtrlEE.ts | 4 +- .../src/lib/version-upgrader/NcUpgrader.ts | 6 +- 83 files changed, 1788 insertions(+), 1117 deletions(-) create mode 100644 packages/nocodb/src/lib/meta/api/projectApis.ts create mode 100644 packages/nocodb/src/lib/services/userService/ui/auth/emailVerify.ts create mode 100644 packages/nocodb/src/lib/services/userService/ui/auth/resetPassword.ts create mode 100644 packages/nocodb/src/lib/services/userService/ui/emailTemplates/forgotPassword.ts create mode 100644 packages/nocodb/src/lib/services/userService/ui/emailTemplates/invite.ts create mode 100644 packages/nocodb/src/lib/services/userService/ui/emailTemplates/verify.ts diff --git a/packages/nocodb-sdk/src/lib/Api.ts b/packages/nocodb-sdk/src/lib/Api.ts index c2197a21be..a1017e4b07 100644 --- a/packages/nocodb-sdk/src/lib/Api.ts +++ b/packages/nocodb-sdk/src/lib/Api.ts @@ -651,7 +651,7 @@ export interface HookReqType { env?: string; event: 'after' | 'before'; operation: 'insert' | 'delete' | 'update'; - async?: string | number | null; + async?: BoolType; notification: object; retries?: number; retry_interval?: number; @@ -1095,8 +1095,8 @@ export class Api< > extends HttpClient { auth = { /** - * @description Create a new user with provided email and password and first user is marked as super admin. - * + * @description Create a new user with provided email and password and first user is marked as super admin. + * * @tags Auth * @name Signup * @summary Signup @@ -1130,7 +1130,7 @@ export class Api< /** * @description Clear refresh token from the database and cookie. - * + * * @tags Auth * @name Signout * @summary Signout @@ -1154,8 +1154,8 @@ export class Api< }), /** - * @description Authenticate existing user with their email and password. Successful login will return a JWT access-token. - * + * @description Authenticate existing user with their email and password. Successful login will return a JWT access-token. + * * @tags Auth * @name Signin * @summary Signin @@ -1231,7 +1231,7 @@ export class Api< /** * @description Change password of authenticated user with a new one. - * + * * @tags Auth * @name PasswordChange * @summary Password change @@ -1334,7 +1334,7 @@ export class Api< /** * No description - * + * * @tags Auth * @name ProjectUserList * @summary Project users @@ -1455,7 +1455,7 @@ export class Api< orgTokens = { /** * No description - * + * * @tags Org tokens * @name List * @summary Organisation API Tokens List @@ -1525,7 +1525,7 @@ export class Api< orgLicense = { /** * No description - * + * * @tags Org license * @name Get * @summary App license get @@ -1569,7 +1569,7 @@ export class Api< orgAppSettings = { /** * No description - * + * * @tags Org app settings * @name Get * @summary App settings get @@ -1618,7 +1618,7 @@ export class Api< orgUsers = { /** * No description - * + * * @tags Org users * @name List * @summary Organisation Users @@ -1723,7 +1723,7 @@ export class Api< /** * No description - * + * * @tags Org users * @name GeneratePasswordResetToken * @summary Organisation User Generate Password Reset Token @@ -1751,7 +1751,7 @@ export class Api< project = { /** * No description - * + * * @tags Project * @name MetaGet * @summary Project info @@ -1936,7 +1936,7 @@ export class Api< /** * @description Read project details - * + * * @tags Project * @name SharedBaseGet * @request GET:/api/v1/db/meta/projects/{projectId}/shared @@ -2004,7 +2004,7 @@ export class Api< /** * No description - * + * * @tags Project * @name SharedBaseUpdate * @request PATCH:/api/v1/db/meta/projects/{projectId}/shared @@ -2106,7 +2106,7 @@ export class Api< /** * No description - * + * * @tags Project * @name AuditList * @request GET:/api/v1/db/meta/projects/{projectId}/audits @@ -2998,7 +2998,7 @@ export class Api< /** * No description - * + * * @tags DB view share * @name Create * @request POST:/api/v1/db/meta/views/{viewId}/share @@ -3117,7 +3117,7 @@ export class Api< dbTableSort = { /** * No description - * + * * @tags DB table sort * @name List * @request GET:/api/v1/db/meta/views/{viewId}/sorts @@ -4318,7 +4318,7 @@ export class Api< /** * @description Read project details - * + * * @tags Public * @name SharedBaseGet * @request GET:/api/v1/db/public/shared-base/{sharedBaseUuid}/meta @@ -4342,7 +4342,7 @@ export class Api< /** * No description - * + * * @tags Public * @name SharedViewMetaGet * @request GET:/api/v1/db/public/shared-view/{sharedViewUuid}/meta @@ -4475,7 +4475,7 @@ export class Api< /** * No description - * + * * @tags Utils * @name TestConnection * @request POST:/api/v1/db/meta/connection/test @@ -4585,7 +4585,7 @@ export class Api< /** * No description - * + * * @tags Utils * @name AggregatedMetaInfo * @request GET:/api/v1/aggregated-meta-info @@ -4702,7 +4702,7 @@ export class Api< dbTableWebhook = { /** * No description - * + * * @tags DB table webhook * @name List * @request GET:/api/v1/db/meta/tables/{tableId}/hooks @@ -4768,7 +4768,7 @@ export class Api< /** * No description - * + * * @tags DB table webhook * @name SamplePayloadGet * @request GET:/api/v1/db/meta/tables/{tableId}/hooks/samplePayload/{operation} @@ -4837,7 +4837,7 @@ export class Api< plugin = { /** * No description - * + * * @tags Plugin * @name List * @request GET:/api/v1/db/meta/plugins diff --git a/packages/nocodb/package-lock.json b/packages/nocodb/package-lock.json index 1f94426e0a..4950e351b1 100644 --- a/packages/nocodb/package-lock.json +++ b/packages/nocodb/package-lock.json @@ -65,7 +65,7 @@ "multer": "^1.4.2", "mysql2": "^2.2.5", "nanoid": "^3.1.20", - "nc-help": "0.2.85", + "nc-help": "0.2.87", "nc-lib-gui": "0.105.3", "nc-plugin": "0.1.2", "ncp": "^2.0.0", @@ -11294,9 +11294,9 @@ } }, "node_modules/nc-help": { - "version": "0.2.85", - "resolved": "https://registry.npmjs.org/nc-help/-/nc-help-0.2.85.tgz", - "integrity": "sha512-EOyrc2PuRUJzv73jHNHmUR6YhC0TlJG0DTY/sug7BF4MJAVPJgyavJnrqkRC7g0NS4xociki9gs5MbLRjlRwtQ==", + "version": "0.2.87", + "resolved": "https://registry.npmjs.org/nc-help/-/nc-help-0.2.87.tgz", + "integrity": "sha512-Zlg06ialvylBEE1qtvjlNKxZrPShzXwvy3WG7nfw+8GngOkQBCTlKguejT2Kq4Gfb5378WPX1APXtsetMKBrRA==", "dependencies": { "@rudderstack/rudder-sdk-node": "^1.1.3", "axios": "^0.21.1", @@ -28004,9 +28004,9 @@ "integrity": "sha512-3AryS9uwa5NfISLxMciUonrH7YfXp+nlahB9T7girXIsLQrmwX4MdnuKs32akduCOGpKmjTJSWmATULbuMkbfw==" }, "nc-help": { - "version": "0.2.85", - "resolved": "https://registry.npmjs.org/nc-help/-/nc-help-0.2.85.tgz", - "integrity": "sha512-EOyrc2PuRUJzv73jHNHmUR6YhC0TlJG0DTY/sug7BF4MJAVPJgyavJnrqkRC7g0NS4xociki9gs5MbLRjlRwtQ==", + "version": "0.2.87", + "resolved": "https://registry.npmjs.org/nc-help/-/nc-help-0.2.87.tgz", + "integrity": "sha512-Zlg06ialvylBEE1qtvjlNKxZrPShzXwvy3WG7nfw+8GngOkQBCTlKguejT2Kq4Gfb5378WPX1APXtsetMKBrRA==", "requires": { "@rudderstack/rudder-sdk-node": "^1.1.3", "axios": "^0.21.1", diff --git a/packages/nocodb/package.json b/packages/nocodb/package.json index 0b77966f58..9682e923c6 100644 --- a/packages/nocodb/package.json +++ b/packages/nocodb/package.json @@ -105,7 +105,7 @@ "multer": "^1.4.2", "mysql2": "^2.2.5", "nanoid": "^3.1.20", - "nc-help": "0.2.85", + "nc-help": "0.2.87", "nc-lib-gui": "0.105.3", "nc-plugin": "0.1.2", "ncp": "^2.0.0", diff --git a/packages/nocodb/src/lib/Noco.ts b/packages/nocodb/src/lib/Noco.ts index 00b80d421b..ca4ae28486 100644 --- a/packages/nocodb/src/lib/Noco.ts +++ b/packages/nocodb/src/lib/Noco.ts @@ -21,7 +21,7 @@ import { NC_LICENSE_KEY } from './constants'; import Migrator from './db/sql-migrator/lib/KnexMigrator'; import Store from './models/Store'; import NcConfigFactory from './utils/NcConfigFactory'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import NcProjectBuilderCE from './v1-legacy/NcProjectBuilder'; import NcProjectBuilderEE from './v1-legacy/NcProjectBuilderEE'; @@ -275,10 +275,10 @@ export default class Noco { } next(); }); - Tele.init({ + T.init({ instance: getInstance, }); - Tele.emit('evt_app_started', await User.count()); + T.emit('evt_app_started', await User.count()); console.log(`App started successfully.\nVisit -> ${Noco.dashboardUrl}`); weAreHiring(); return this.router; @@ -531,7 +531,7 @@ export default class Noco { if (!serverId) { await Noco._ncMeta.metaInsert('', '', 'nc_store', { key: 'nc_server_id', - value: (serverId = Tele.id), + value: (serverId = T.id), }); } process.env.NC_SERVER_UUID = serverId; diff --git a/packages/nocodb/src/lib/controllers/columnController.ts b/packages/nocodb/src/lib/controllers/columnController.ts index dad4a4c01b..61f2e0139d 100644 --- a/packages/nocodb/src/lib/controllers/columnController.ts +++ b/packages/nocodb/src/lib/controllers/columnController.ts @@ -1,8 +1,8 @@ -import { Request, Response, Router } from 'express' +import { Request, Response, Router } from 'express'; import { ColumnReqType, TableType, UITypes } from 'nocodb-sdk'; -import { getAjvValidatorMw } from '../meta/api/helpers' -import { metaApiMetrics } from '../meta/helpers/apiMetrics' -import ncMetaAclMw from '../meta/helpers/ncMetaAclMw' +import { getAjvValidatorMw } from '../meta/api/helpers'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { columnService } from '../services'; export async function columnGet(req: Request, res: Response) { @@ -27,20 +27,19 @@ export async function columnSetAsPrimary(req: Request, res: Response) { ); } - - -export async function columnUpdate(req: Request, res: Response){ - res.json(await columnService.columnUpdate({ - columnId: req.params.columnId, - column: req.body - }) +export async function columnUpdate(req: Request, res: Response) { + res.json( + await columnService.columnUpdate({ + columnId: req.params.columnId, + column: req.body, + }) + ); } export async function columnDelete(req: Request, res: Response) { - res.json(await columnService.columnDelete({ columnId: req.params.columnId })); + res.json(await columnService.columnDelete({ columnId: req.params.columnId })); } - const router = Router({ mergeParams: true }); router.post( diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts index 8593ff6fa9..f0f145f596 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts @@ -15,7 +15,7 @@ async function excelDataExport(req: Request, res: Response) { if (!targetView) { targetView = await View.getDefaultView(model.id); } - const { offset, elapsed, data } = await extractXlsxData(targetView, req); + const { offset, elapsed, data } = await extractXlsxData({ view: targetView, query: req.query, siteUrl: (req as any).ncSiteUrl }); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, data, targetView.title); const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); diff --git a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts index ed48507e0d..1a7216f5fc 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts @@ -1,20 +1,19 @@ -import Project from '../../models/Project'; -import Model from '../../models/Model'; -import View from '../../models/View'; -import { NcError } from '../../meta/helpers/catchError'; +import Project from '../../../models/Project'; +import Model from '../../../models/Model'; +import View from '../../../models/View'; +import { NcError } from '../../helpers/catchError'; import { Request } from 'express'; -import Base from '../../models/Base'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import Base from '../../../models/Base'; +import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; import { isSystemColumn, UITypes } from 'nocodb-sdk'; -import { nocoExecute } from 'nc-help'; import * as XLSX from 'xlsx'; -import Column from '../../models/Column'; -import LookupColumn from '../../models/LookupColumn'; -import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; +import Column from '../../../models/Column'; +import LookupColumn from '../../../models/LookupColumn'; +import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; import papaparse from 'papaparse'; -import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import { dataService } from '../../../services'; export async function getViewAndModelFromRequestByAliasOrId( req: | Request<{ projectName: string; tableName: string; viewName?: string }> @@ -36,8 +35,12 @@ export async function getViewAndModelFromRequestByAliasOrId( return { model, view }; } -export async function extractXlsxData(param: {view: View, query:any}) { - const { view, query } = param; +export async function extractXlsxData(param: { + view: View; + query: any; + siteUrl: string; +}) { + const { view, query, siteUrl } = param; const base = await Base.get(view.base_id); await view.getModelWithInfo(); @@ -57,9 +60,14 @@ export async function extractXlsxData(param: {view: View, query:any}) { dbDriver: NcConnectionMgrv2.get(base), }); - const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req); + const { offset, dbRows, elapsed } = await dataService.getDbRows({ + baseModel, + view, + query, + siteUrl, + }); - const fields = req.query.fields as string[]; + const fields = query.fields as string[]; const data = XLSX.utils.json_to_sheet(dbRows, { header: fields }); @@ -87,7 +95,12 @@ export async function extractCsvData(view: View, req: Request) { dbDriver: NcConnectionMgrv2.get(base), }); - const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req); + const { offset, dbRows, elapsed } = await dataService.getDbRows({ + baseModel, + view, + query: req.query, + siteUrl: (req as any).ncSiteUrl, + }); const data = papaparse.unparse( { @@ -111,64 +124,64 @@ export async function extractCsvData(view: View, req: Request) { return { offset, dbRows, elapsed, data }; } - -async function getDbRows(baseModel, view: View, req: Request) { - let offset = +req.query.offset || 0; - const limit = 100; - // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; - const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; - const dbRows = []; - const startTime = process.hrtime(); - let elapsed, temp; - - const listArgs: any = { ...req.query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - - for ( - elapsed = 0; - elapsed < timeout; - offset += limit, - temp = process.hrtime(startTime), - elapsed = temp[0] * 1000 + temp[1] / 1000000 - ) { - const rows = await nocoExecute( - await getAst({ - query: req.query, - includePkByDefault: false, - model: view.model, - view, - }), - await baseModel.list({ ...listArgs, offset, limit }), - {}, - req.query - ); - - if (!rows?.length) { - offset = -1; - break; - } - - for (const row of rows) { - const dbRow = { ...row }; - - for (const column of view.model.columns) { - if (isSystemColumn(column) && !view.show_system_fields) continue; - dbRow[column.title] = await serializeCellValue({ - value: row[column.title], - column, - siteUrl: req['ncSiteUrl'], - }); - } - dbRows.push(dbRow); - } - } - return { offset, dbRows, elapsed }; -} +// +// async function getDbRows(baseModel, view: View, req: Request) { +// let offset = +req.query.offset || 0; +// const limit = 100; +// // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; +// const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; +// const dbRows = []; +// const startTime = process.hrtime(); +// let elapsed, temp; +// +// const listArgs: any = { ...req.query }; +// try { +// listArgs.filterArr = JSON.parse(listArgs.filterArrJson); +// } catch (e) {} +// try { +// listArgs.sortArr = JSON.parse(listArgs.sortArrJson); +// } catch (e) {} +// +// for ( +// elapsed = 0; +// elapsed < timeout; +// offset += limit, +// temp = process.hrtime(startTime), +// elapsed = temp[0] * 1000 + temp[1] / 1000000 +// ) { +// const rows = await nocoExecute( +// await getAst({ +// query: req.query, +// includePkByDefault: false, +// model: view.model, +// view, +// }), +// await baseModel.list({ ...listArgs, offset, limit }), +// {}, +// req.query +// ); +// +// if (!rows?.length) { +// offset = -1; +// break; +// } +// +// for (const row of rows) { +// const dbRow = { ...row }; +// +// for (const column of view.model.columns) { +// if (isSystemColumn(column) && !view.show_system_fields) continue; +// dbRow[column.title] = await serializeCellValue({ +// value: row[column.title], +// column, +// siteUrl: req['ncSiteUrl'], +// }); +// } +// dbRows.push(dbRow); +// } +// } +// return { offset, dbRows, elapsed }; +// } export async function serializeCellValue({ value, diff --git a/packages/nocodb/src/lib/controllers/dataController/export.ts b/packages/nocodb/src/lib/controllers/dataController/export.ts index 8c6cf38bea..05f77dde5c 100644 --- a/packages/nocodb/src/lib/controllers/dataController/export.ts +++ b/packages/nocodb/src/lib/controllers/dataController/export.ts @@ -1,13 +1,13 @@ import { Request, Response, Router } from 'express'; import * as XLSX from 'xlsx'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import { getViewAndModelFromRequestByAliasOrId } from '../../meta/api/dataApis/helpers'; +import apiMetrics from '../../meta/helpers/apiMetrics'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import { View } from '../../models'; import { extractCsvData, extractXlsxData, - getViewAndModelFromRequestByAliasOrId, -} from './helpers'; -import apiMetrics from '../../helpers/apiMetrics'; -import View from '../../../models/View'; +} from '../../services/dataService/helpers'; async function excelDataExport(req: Request, res: Response) { const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); diff --git a/packages/nocodb/src/lib/controllers/hookFilterController.ts b/packages/nocodb/src/lib/controllers/hookFilterController.ts index 0d0aa0a8da..71e3977802 100644 --- a/packages/nocodb/src/lib/controllers/hookFilterController.ts +++ b/packages/nocodb/src/lib/controllers/hookFilterController.ts @@ -1,112 +1,60 @@ import { Request, Response, Router } from 'express'; -// @ts-ignore -import Model from '../models/Model'; -import { Tele } from 'nc-help'; -// @ts-ignore -import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -// @ts-ignore -import { Table, TableList, TableListParams, TableReq } from 'nocodb-sdk'; -// @ts-ignore -import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; -// @ts-ignore -import Project from '../models/Project'; -import Filter from '../models/Filter'; +import { T } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import { getAjvValidatorMw } from '../meta/api/helpers'; +import { hookFilterService } from '../services'; -// @ts-ignore -export async function filterGet(req: Request, res: Response, next) { - try { - const filter = await Filter.getFilterObject({ hookId: req.params.hookId }); +export async function filterGet(req: Request, res: Response) { + const filter = await hookFilterService.filterGet({ + hookId: req.params.hookId, + }); - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } + res.json(filter); } -// @ts-ignore -export async function filterList( - req: Request, - res: Response, - next -) { - try { - const filter = await Filter.rootFilterListByHook({ - hookId: req.params.hookId, - }); +export async function filterList(req: Request, res: Response) { + const filter = await hookFilterService.filterList({ + hookId: req.params.hookId, + }); - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } + res.json(filter); } -// @ts-ignore -export async function filterChildrenRead( - req: Request, - res: Response, - next -) { - try { - const filter = await Filter.parentFilterListByHook({ - hookId: req.params.hookId, - parentId: req.params.filterParentId, - }); - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } +export async function filterChildrenRead(req: Request, res: Response) { + const filter = await hookFilterService.filterChildrenRead({ + hookId: req.params.hookId, + filterParentId: req.params.filterParentId, + }); + + res.json(filter); } -export async function filterCreate( - req: Request, - res, - next -) { - try { - const filter = await Filter.insert({ - ...req.body, - fk_hook_id: req.params.hookId, - }); +export async function filterCreate(req: Request, res) { + const filter = await hookFilterService.filterCreate({ + filter: req.body, + hookId: req.params.hookId, + }); - Tele.emit('evt', { evt_type: 'hookFilter:created' }); - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } + res.json(filter); } -// @ts-ignore -export async function filterUpdate(req, res, next) { - try { - const filter = await Filter.update(req.params.filterId, { - ...req.body, - fk_hook_id: req.params.hookId, - }); - Tele.emit('evt', { evt_type: 'hookFilter:updated' }); - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } +export async function filterUpdate(req, res) { + const filter = await hookFilterService.filterUpdate({ + filterId: req.params.filterId, + filter: req.body, + hookId: req.params.hookId, + }); + + res.json(filter); } -// @ts-ignore -export async function filterDelete(req: Request, res: Response, next) { - try { - const filter = await Filter.delete(req.params.filterId); - Tele.emit('evt', { evt_type: 'hookFilter:deleted' }); - res.json(filter); - } catch (e) { - console.log(e); - next(e); - } +export async function filterDelete(req: Request, res: Response) { + const filter = await hookFilterService.filterDelete({ + filterId: req.params.filterId, + }); + T.emit('evt', { evt_type: 'hookFilter:deleted' }); + res.json(filter); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/kanbanViewController.ts b/packages/nocodb/src/lib/controllers/kanbanViewController.ts index e9db2a3afb..7ea705e6bb 100644 --- a/packages/nocodb/src/lib/controllers/kanbanViewController.ts +++ b/packages/nocodb/src/lib/controllers/kanbanViewController.ts @@ -2,7 +2,7 @@ import { Request, Response, Router } from 'express'; import { KanbanType, ViewTypes } from 'nocodb-sdk'; import View from '../models/View'; import KanbanView from '../models/KanbanView'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import { getAjvValidatorMw } from '../meta/api/helpers'; @@ -12,7 +12,7 @@ export async function kanbanViewGet(req: Request, res: Response) { } export async function kanbanViewCreate(req: Request, res) { - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'kanban' }); + T.emit('evt', { evt_type: 'vtable:created', show_as: 'kanban' }); const view = await View.insert({ ...req.body, // todo: sanitize @@ -23,7 +23,7 @@ export async function kanbanViewCreate(req: Request, res) { } export async function kanbanViewUpdate(req, res) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'kanban' }); + T.emit('evt', { evt_type: 'view:updated', type: 'kanban' }); res.json(await KanbanView.update(req.params.kanbanViewId, req.body)); } diff --git a/packages/nocodb/src/lib/controllers/orgLicenseController.ts b/packages/nocodb/src/lib/controllers/orgLicenseController.ts index 02b167c3df..de6d839537 100644 --- a/packages/nocodb/src/lib/controllers/orgLicenseController.ts +++ b/packages/nocodb/src/lib/controllers/orgLicenseController.ts @@ -10,7 +10,7 @@ async function licenseGet(_req, res) { } async function licenseSet(req, res) { - await orgLicenseService.licenseSet({ key: req.body.key }) + await orgLicenseService.licenseSet({ key: req.body.key }); res.json({ msg: 'License key saved' }); } diff --git a/packages/nocodb/src/lib/controllers/pluginController.ts b/packages/nocodb/src/lib/controllers/pluginController.ts index c5bef7e9a0..156fce97db 100644 --- a/packages/nocodb/src/lib/controllers/pluginController.ts +++ b/packages/nocodb/src/lib/controllers/pluginController.ts @@ -6,8 +6,6 @@ import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import { getAjvValidatorMw } from '../meta/api/helpers'; import { pluginService } from '../services'; - - export async function pluginList(_req: Request, res: Response) { res.json(new PagedResponseImpl(await pluginService.pluginList())); } @@ -23,11 +21,16 @@ export async function pluginUpdate( req: Request, res: Response ) { - const plugin = await pluginService.pluginUpdate({ pluginId: req.params.pluginId, plugin:req.body }); + const plugin = await pluginService.pluginUpdate({ + pluginId: req.params.pluginId, + plugin: req.body, + }); res.json(plugin); } export async function isPluginActive(req: Request, res: Response) { - res.json(await pluginService.isPluginActive({ pluginTitle: req.params.pluginTitle })); + res.json( + await pluginService.isPluginActive({ pluginTitle: req.params.pluginTitle }) + ); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/projectController.ts b/packages/nocodb/src/lib/controllers/projectController.ts index 5e65b8efdd..1be862a7e4 100644 --- a/packages/nocodb/src/lib/controllers/projectController.ts +++ b/packages/nocodb/src/lib/controllers/projectController.ts @@ -3,7 +3,7 @@ import { ProjectType } from 'nocodb-sdk'; import Project from '../models/Project'; import { ProjectListType } from 'nocodb-sdk'; import { packageVersion } from '../utils/packageVersion'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; @@ -60,10 +60,7 @@ export async function projectList( ); } -export async function projectDelete( - req: Request, - res: Response -) { +export async function projectDelete(req: Request, res: Response) { const deleted = await projectService.projectSoftDelete({ projectId: req.params.projectId, }); @@ -71,7 +68,6 @@ export async function projectDelete( res.json(deleted); } - async function projectCreate(req: Request, res) { const project = await projectService.projectCreate({ project: req.body, @@ -116,7 +112,7 @@ export async function projectCost(req, res) { } } - Tele.event({ + T.event({ event: 'a:project:cost', data: { cost, diff --git a/packages/nocodb/src/lib/controllers/public/index.ts b/packages/nocodb/src/lib/controllers/public/index.ts index 60e7849dc2..8e1c237b36 100644 --- a/packages/nocodb/src/lib/controllers/public/index.ts +++ b/packages/nocodb/src/lib/controllers/public/index.ts @@ -2,4 +2,8 @@ import publicDataController from './publicDataApis'; import publicDataExportController from './publicDataExportApis'; import publicMetaController from './publicMetaApis'; -export { publicDataController, publicDataExportController, publicMetaController }; +export { + publicDataController, + publicDataExportController, + publicMetaController, +}; diff --git a/packages/nocodb/src/lib/controllers/sortController.ts b/packages/nocodb/src/lib/controllers/sortController.ts index c9ee69515a..397b605f57 100644 --- a/packages/nocodb/src/lib/controllers/sortController.ts +++ b/packages/nocodb/src/lib/controllers/sortController.ts @@ -1,5 +1,5 @@ import { Request, Response, Router } from 'express'; -import { getAjvValidatorMw } from '../meta/api/helpers' +import { getAjvValidatorMw } from '../meta/api/helpers'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import { SortListType, SortReqType } from 'nocodb-sdk'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; diff --git a/packages/nocodb/src/lib/controllers/swaggerController/index.ts b/packages/nocodb/src/lib/controllers/swaggerController/index.ts index 1dba08f86f..1853e4c7ff 100644 --- a/packages/nocodb/src/lib/controllers/swaggerController/index.ts +++ b/packages/nocodb/src/lib/controllers/swaggerController/index.ts @@ -1,36 +1,36 @@ -import { Router } from 'express' -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw' -import getSwaggerHtml from './swaggerHtml' -import getRedocHtml from './redocHtml' -import { swaggerService } from '../../services' +import { Router } from 'express'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import getSwaggerHtml from './swaggerHtml'; +import getRedocHtml from './redocHtml'; +import { swaggerService } from '../../services'; async function swaggerJson(req, res) { const swagger = await swaggerService.swaggerJson({ projectId: req.params.projectId, siteUrl: req.ncSiteUrl, - }) + }); - res.json(swagger) + res.json(swagger); } function swaggerHtml(_, res) { - res.send(getSwaggerHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' })) + res.send(getSwaggerHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' })); } function redocHtml(_, res) { - res.send(getRedocHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' })) + res.send(getRedocHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' })); } -const router = Router({ mergeParams: true }) +const router = Router({ mergeParams: true }); // todo: auth router.get( '/api/v1/db/meta/projects/:projectId/swagger.json', - ncMetaAclMw(swaggerJson, 'swaggerJson'), -) + ncMetaAclMw(swaggerJson, 'swaggerJson') +); -router.get('/api/v1/db/meta/projects/:projectId/swagger', swaggerHtml) +router.get('/api/v1/db/meta/projects/:projectId/swagger', swaggerHtml); -router.get('/api/v1/db/meta/projects/:projectId/redoc', redocHtml) +router.get('/api/v1/db/meta/projects/:projectId/redoc', redocHtml); -export default router +export default router; diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/job.ts b/packages/nocodb/src/lib/controllers/sync/helpers/job.ts index 2b4364ca05..9e55beb4bd 100644 --- a/packages/nocodb/src/lib/controllers/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/controllers/sync/helpers/job.ts @@ -1,4 +1,4 @@ -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import FetchAT from './fetchAT'; import { UITypes } from 'nocodb-sdk'; // import * as sMap from './syncMap'; @@ -1924,7 +1924,7 @@ export default async ( }); } - Tele.event({ + T.event({ event: 'a:airtable-import:success', data: { stats: { @@ -2391,7 +2391,7 @@ export default async ( } } catch (e) { if (e.response?.data?.msg) { - Tele.event({ + T.event({ event: 'a:airtable-import:error', data: { error: e.response.data.msg }, }); diff --git a/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts b/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts index 2720f8fd88..9da8d70a70 100644 --- a/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts +++ b/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts @@ -1,10 +1,10 @@ import { Request, Response, Router } from 'express'; -import SyncSource from '../../models/SyncSource'; -import { Tele } from 'nc-help'; -import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import Project from '../../models/Project'; +import SyncSource from '../../../models/SyncSource'; +import { T } from 'nc-help'; +import { PagedResponseImpl } from '../../helpers/PagedResponse'; +import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import Project from '../../../models/Project'; export async function syncSourceList(req: Request, res: Response) { // todo: pagination @@ -16,7 +16,7 @@ export async function syncSourceList(req: Request, res: Response) { } export async function syncCreate(req: Request, res: Response) { - Tele.emit('evt', { evt_type: 'webhooks:created' }); + T.emit('evt', { evt_type: 'webhooks:created' }); const project = await Project.getWithInfo(req.params.projectId); const sync = await SyncSource.insert({ @@ -29,12 +29,12 @@ export async function syncCreate(req: Request, res: Response) { } export async function syncDelete(req: Request, res: Response) { - Tele.emit('evt', { evt_type: 'webhooks:deleted' }); + T.emit('evt', { evt_type: 'webhooks:deleted' }); res.json(await SyncSource.delete(req.params.syncId)); } export async function syncUpdate(req: Request, res: Response) { - Tele.emit('evt', { evt_type: 'webhooks:updated' }); + T.emit('evt', { evt_type: 'webhooks:updated' }); res.json(await SyncSource.update(req.params.syncId, req.body)); } diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index 8ee37c019f..df80818bff 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -42,7 +42,7 @@ export async function tableGet(req: Request, res: Response) { } export async function tableDelete(req: Request, res: Response) { - const result = await tableService.deleteTable({ + const result = await tableService.tableDelete({ tableId: req.params.tableId, user: (req as any).session?.passport?.user, }); diff --git a/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts b/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts index c39328bfb2..185fd155ae 100644 --- a/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts +++ b/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts @@ -1,15 +1,15 @@ -import User from '../../models/User'; +import User from '../../../models/User'; import { v4 as uuidv4 } from 'uuid'; import { promisify } from 'util'; import bcrypt from 'bcryptjs'; -import Noco from '../../Noco'; -import { CacheScope, MetaTable } from '../../utils/globals'; -import ProjectUser from '../../models/ProjectUser'; +import Noco from '../../../Noco'; +import { CacheScope, MetaTable } from '../../../utils/globals'; +import ProjectUser from '../../../models/ProjectUser'; import { validatePassword } from 'nocodb-sdk'; import boxen from 'boxen'; -import NocoCache from '../../cache/NocoCache'; -import { Tele } from 'nc-help'; +import NocoCache from '../../../cache/NocoCache'; +import { T } from 'nc-help'; const { isEmail } = require('validator'); const rolesLevel = { owner: 0, creator: 1, editor: 2, commenter: 3, viewer: 4 }; @@ -68,7 +68,7 @@ export default async function initAdminFromEnv(_ncMeta = Noco.ncMeta) { // if super admin not present if (await User.isFirst(ncMeta)) { // roles = 'owner,creator,editor' - Tele.emit('evt', { + T.emit('evt', { evt_type: 'project:invite', count: 1, }); @@ -126,7 +126,7 @@ export default async function initAdminFromEnv(_ncMeta = Noco.ncMeta) { ncMeta ); } else { - Tele.emit('evt', { + T.emit('evt', { evt_type: 'project:invite', count: 1, }); diff --git a/packages/nocodb/src/lib/controllers/userApi/userApis.ts b/packages/nocodb/src/lib/controllers/userApi/userApis.ts index b41ca7dfa1..fa35a5bd0d 100644 --- a/packages/nocodb/src/lib/controllers/userApi/userApis.ts +++ b/packages/nocodb/src/lib/controllers/userApi/userApis.ts @@ -1,30 +1,30 @@ import { Request, Response } from 'express'; import { TableType, validatePassword } from 'nocodb-sdk'; import { OrgUserRoles } from 'nocodb-sdk'; -import { NC_APP_SETTINGS } from '../../constants'; -import Store from '../../models/Store'; -import { Tele } from 'nc-help'; -import catchError, { NcError } from '../../meta/helpers/catchError'; +import { NC_APP_SETTINGS } from '../../../constants'; +import Store from '../../../models/Store'; +import { T } from 'nc-help'; +import catchError, { NcError } from '../../helpers/catchError'; const { isEmail } = require('validator'); import * as ejs from 'ejs'; import bcrypt from 'bcryptjs'; import { promisify } from 'util'; -import User from '../../models/User'; +import User from '../../../models/User'; const { v4: uuidv4 } = require('uuid'); -import Audit from '../../models/Audit'; -import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; +import Audit from '../../../models/Audit'; +import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2'; import passport from 'passport'; -import extractProjectIdAndAuthenticate from '../../meta/helpers/extractProjectIdAndAuthenticate'; -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import { MetaTable } from '../../utils/globals'; -import Noco from '../../Noco'; -import { getAjvValidatorMw } from '../../meta/api/helpers'; +import extractProjectIdAndAuthenticate from '../../helpers/extractProjectIdAndAuthenticate'; +import ncMetaAclMw from '../../helpers/ncMetaAclMw'; +import { MetaTable } from '../../../utils/globals'; +import Noco from '../../../Noco'; +import { getAjvValidatorMw } from '../helpers'; import { genJwt } from './helpers'; -import { randomTokenString } from '../../meta/helpers/stringHelpers'; +import { randomTokenString } from '../../helpers/stringHelpers'; export async function registerNewUserIfAllowed({ firstname, @@ -47,7 +47,7 @@ export async function registerNewUserIfAllowed({ roles = `${OrgUserRoles.CREATOR},${OrgUserRoles.SUPER_ADMIN}`; // todo: update in nc_store // roles = 'owner,creator,editor' - Tele.emit('evt', { + T.emit('evt', { evt_type: 'project:invite', count: 1, }); @@ -122,7 +122,7 @@ export async function signup(req: Request, res: Response) { const email_verification_token = uuidv4(); if (!ignore_subscribe) { - Tele.emit('evt_subscribe', email); + T.emit('evt_subscribe', email); } if (user) { diff --git a/packages/nocodb/src/lib/controllers/userController/index.ts b/packages/nocodb/src/lib/controllers/userController/index.ts index 0b42c7c9c3..70d17b40ef 100644 --- a/packages/nocodb/src/lib/controllers/userController/index.ts +++ b/packages/nocodb/src/lib/controllers/userController/index.ts @@ -1,6 +1,6 @@ import { Request, Response } from 'express'; import { TableType, validatePassword } from 'nocodb-sdk'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; const { isEmail } = require('validator'); import * as ejs from 'ejs'; @@ -64,7 +64,7 @@ export async function signup(req: Request, res: Response) { const email_verification_token = uuidv4(); if (!ignore_subscribe) { - Tele.emit('evt_subscribe', email); + T.emit('evt_subscribe', email); } if (user) { diff --git a/packages/nocodb/src/lib/controllers/viewController.ts b/packages/nocodb/src/lib/controllers/viewController.ts index e0a1cc6631..edbc2ed736 100644 --- a/packages/nocodb/src/lib/controllers/viewController.ts +++ b/packages/nocodb/src/lib/controllers/viewController.ts @@ -9,10 +9,7 @@ import { viewService } from '../services'; export async function viewGet(req: Request, res: Response) {} // @ts-ignore -export async function viewList( - req: Request, - res: Response -) { +export async function viewList(req: Request, res: Response) { const filteredViewList = await viewService.viewList({ tableId: req.params.tableId, user: (req as any).session?.passport?.user, diff --git a/packages/nocodb/src/lib/db/sql-client/lib/KnexClient.ts b/packages/nocodb/src/lib/db/sql-client/lib/KnexClient.ts index 9c47ab34ac..7643c92c9b 100644 --- a/packages/nocodb/src/lib/db/sql-client/lib/KnexClient.ts +++ b/packages/nocodb/src/lib/db/sql-client/lib/KnexClient.ts @@ -1,6 +1,6 @@ /* eslint-disable no-constant-condition */ import { knex, Knex } from 'knex'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import Debug from '../../util/Debug'; import Emit from '../../util/emit'; import Result from '../../util/Result'; @@ -620,7 +620,7 @@ class KnexClient extends SqlClient { KnexClient.___ext = await this._validateInput(); } if (!KnexClient.___ext) { - Tele.emit('evt', { + T.emit('evt', { evt_type: 'project:external', payload: null, check: true, diff --git a/packages/nocodb/src/lib/db/sql-mgr/SqlMgr.ts b/packages/nocodb/src/lib/db/sql-mgr/SqlMgr.ts index 3f6a85b9c2..55ea1559d7 100644 --- a/packages/nocodb/src/lib/db/sql-mgr/SqlMgr.ts +++ b/packages/nocodb/src/lib/db/sql-mgr/SqlMgr.ts @@ -7,7 +7,7 @@ import fsExtra from 'fs-extra'; import importFresh from 'import-fresh'; import inflection from 'inflection'; import slash from 'slash'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import SqlClientFactory from '../sql-client/lib/SqlClientFactory'; // import debug from 'debug'; @@ -1047,7 +1047,7 @@ export default class SqlMgr { // t = process.hrtime(); const data = await require('axios')(...apiMeta); - Tele.emit('evt', { evt_type: 'import:excel:url' }); + T.emit('evt', { evt_type: 'import:excel:url' }); return data.data; } diff --git a/packages/nocodb/src/lib/meta/NcMetaMgr.ts b/packages/nocodb/src/lib/meta/NcMetaMgr.ts index aecaff7189..da98f2c5e9 100644 --- a/packages/nocodb/src/lib/meta/NcMetaMgr.ts +++ b/packages/nocodb/src/lib/meta/NcMetaMgr.ts @@ -40,7 +40,7 @@ import NcTemplateParser from '../v1-legacy/templates/NcTemplateParser'; import { defaultConnectionConfig } from '../utils/NcConfigFactory'; import xcMetaDiff from './handlers/xcMetaDiff'; import { UITypes } from 'nocodb-sdk'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { NC_ATTACHMENT_FIELD_SIZE } from '../constants'; const randomID = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 10); const XC_PLUGIN_DET = 'XC_PLUGIN_DET'; @@ -953,7 +953,7 @@ export default class NcMetaMgr { ip: req.clientIp, }); - Tele.emit('evt', { evt_type: 'webhooks:deleted' }); + T.emit('evt', { evt_type: 'webhooks:deleted' }); } catch (e) { throw e; } @@ -988,7 +988,7 @@ export default class NcMetaMgr { ip: req.clientIp, }); - Tele.emit('evt', { evt_type: 'webhooks:updated' }); + T.emit('evt', { evt_type: 'webhooks:updated' }); } else { const res = await this.xcMeta.metaInsert( projectId, @@ -1009,7 +1009,7 @@ export default class NcMetaMgr { description: `created webhook ${args.args.data.title} - ${args.args.data.event} ${args.args.data.operation} - ${args.args.data.notification?.type} - of table ${args.args.tn} `, ip: req.clientIp, }); - Tele.emit('evt', { evt_type: 'webhooks:created' }); + T.emit('evt', { evt_type: 'webhooks:created' }); return res; } @@ -1269,7 +1269,7 @@ export default class NcMetaMgr { } catch (e) { throw e; } finally { - Tele.emit('evt', { evt_type: 'image:uploaded' }); + T.emit('evt', { evt_type: 'image:uploaded' }); } } @@ -1595,7 +1595,7 @@ export default class NcMetaMgr { ip: req.clientIp, }); - Tele.emit('evt', { evt_type: 'project:created' }); + T.emit('evt', { evt_type: 'project:created' }); break; case 'projectUpdateByWeb': @@ -1603,7 +1603,7 @@ export default class NcMetaMgr { this.getProjectId(args), args.args.projectJson ); - Tele.emit('evt', { evt_type: 'project:updated' }); + T.emit('evt', { evt_type: 'project:updated' }); break; case 'projectCreateByOneClick': { @@ -1635,7 +1635,7 @@ export default class NcMetaMgr { description: `created project ${config.title}(${result.id}) `, ip: req.clientIp, }); - Tele.emit('evt', { evt_type: 'project:created', oneClick: true }); + T.emit('evt', { evt_type: 'project:created', oneClick: true }); } break; case 'projectCreateByWebWithXCDB': { @@ -1689,10 +1689,10 @@ export default class NcMetaMgr { ip: req.clientIp, }); - Tele.emit('evt', { evt_type: 'project:created', xcdb: true }); + T.emit('evt', { evt_type: 'project:created', xcdb: true }); postListenerCb = async () => { if (args?.args?.template) { - Tele.emit('evt', { + T.emit('evt', { evt_type: args.args?.quickImport ? 'project:created:fromExcel' : 'project:created:fromTemplate', @@ -1730,7 +1730,7 @@ export default class NcMetaMgr { case 'projectDelete': case 'projectRestart': case 'projectStart': - Tele.emit('evt', { evt_type: 'project:' + args.api }); + T.emit('evt', { evt_type: 'project:' + args.api }); result = null; break; @@ -2145,7 +2145,7 @@ export default class NcMetaMgr { ip: req.clientIp, }); - Tele.emit('evt', { evt_type: 'acl:updated' }); + T.emit('evt', { evt_type: 'acl:updated' }); return res; } catch (e) { @@ -3485,7 +3485,7 @@ export default class NcMetaMgr { ['id', 'view_id', 'view_type'] ); res.url = `${req.ncSiteUrl}${this.config.dashboardPath}#/nc/view/${res.view_id}`; - Tele.emit('evt', { evt_type: 'sharedView:generated-link' }); + T.emit('evt', { evt_type: 'sharedView:generated-link' }); return res; } catch (e) { console.log(e); @@ -3549,7 +3549,7 @@ export default class NcMetaMgr { sharedBase.url = `${req.ncSiteUrl}${this.config.dashboardPath}#/nc/base/${sharedBase.shared_base_id}`; - Tele.emit('evt', { evt_type: 'sharedBase:generated-link' }); + T.emit('evt', { evt_type: 'sharedBase:generated-link' }); return sharedBase; } catch (e) { console.log(e); @@ -3606,7 +3606,7 @@ export default class NcMetaMgr { // await this.xcMeta.metaUpdate(this.getProjectId(args), this.getDbAlias(args), 'nc_shared_views', { // password: args.args?.password // }, args.args.id); - // Tele.emit('evt', {evt_type: 'sharedView:password-updated'}) + // T.emit('evt', {evt_type: 'sharedView:password-updated'}) // return {msg: 'Success'}; // } catch (e) { // console.log(e) @@ -3623,7 +3623,7 @@ export default class NcMetaMgr { 'nc_shared_views', args.args.id ); - Tele.emit('evt', { evt_type: 'sharedView:deleted' }); + T.emit('evt', { evt_type: 'sharedView:deleted' }); return { msg: 'Success' }; } catch (e) { console.log(e); @@ -4477,7 +4477,7 @@ export default class NcMetaMgr { }); } - Tele.emit('evt', { evt_type: 'template:imported' }); + T.emit('evt', { evt_type: 'template:imported' }); return result; } @@ -5071,7 +5071,7 @@ export default class NcMetaMgr { } catch (e) { throw e; } finally { - Tele.emit('evt', { + T.emit('evt', { evt_type: 'plugin:installed', title: args.args.title, }); @@ -5153,7 +5153,7 @@ export default class NcMetaMgr { ip: req.clientIp, }); - Tele.emit('evt', { + T.emit('evt', { evt_type: 'vtable:created', show_as: args.args.show_as, }); @@ -5270,7 +5270,7 @@ export default class NcMetaMgr { }); await RestAuthCtrl.instance.loadLatestApiTokens(); - Tele.emit('evt', { evt_type: 'apiToken:created' }); + T.emit('evt', { evt_type: 'apiToken:created' }); return { description: args.args.description, token, @@ -5282,7 +5282,7 @@ export default class NcMetaMgr { } protected async xcApiTokenDelete(args): Promise { - Tele.emit('evt', { evt_type: 'apiToken:deleted' }); + T.emit('evt', { evt_type: 'apiToken:deleted' }); const res = await this.xcMeta.metaDelete( null, null, @@ -5327,7 +5327,7 @@ export default class NcMetaMgr { ip: req.clientIp, }); - Tele.emit('evt', { + T.emit('evt', { evt_type: 'vtable:renamed', show_as: args.args.show_as, }); @@ -5335,7 +5335,7 @@ export default class NcMetaMgr { } protected async xcVirtualTableUpdate(args): Promise { - // Tele.emit('evt', {evt_type: 'vtable:updated',show_as: args.args.show_as}) + // T.emit('evt', {evt_type: 'vtable:updated',show_as: args.args.show_as}) return this.xcMeta.metaUpdate( this.getProjectId(args), this.getDbAlias(args), @@ -5482,7 +5482,7 @@ export default class NcMetaMgr { ip: req.clientIp, }); - Tele.emit('evt', { evt_type: 'vtable:deleted' }); + T.emit('evt', { evt_type: 'vtable:deleted' }); return res; } diff --git a/packages/nocodb/src/lib/meta/NcMetaMgrEE.ts b/packages/nocodb/src/lib/meta/NcMetaMgrEE.ts index dfa8dff104..c0e10ec8ab 100644 --- a/packages/nocodb/src/lib/meta/NcMetaMgrEE.ts +++ b/packages/nocodb/src/lib/meta/NcMetaMgrEE.ts @@ -1,5 +1,5 @@ import { v4 as uuidv4 } from 'uuid'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import NcMetaMgr from './NcMetaMgr'; @@ -104,7 +104,7 @@ export default class NcMetaMgrEE extends NcMetaMgr { ip: req.clientIp, }); - Tele.emit('evt', { evt_type: 'acl:updated' }); + T.emit('evt', { evt_type: 'acl:updated' }); return res; } catch (e) { @@ -277,7 +277,7 @@ export default class NcMetaMgrEE extends NcMetaMgr { sharedView.url = `${req.ncSiteUrl}${this.config.dashboardPath}#/nc/view/${sharedView.view_id}`; } - Tele.emit('evt', { evt_type: 'sharedView:generated-link' }); + T.emit('evt', { evt_type: 'sharedView:generated-link' }); return sharedView; } catch (e) { console.log(e); @@ -295,7 +295,7 @@ export default class NcMetaMgrEE extends NcMetaMgr { }, args.args.id ); - Tele.emit('evt', { evt_type: 'sharedView:password-updated' }); + T.emit('evt', { evt_type: 'sharedView:password-updated' }); return { msg: 'Success' }; } catch (e) { console.log(e); diff --git a/packages/nocodb/src/lib/meta/api/projectApis.ts b/packages/nocodb/src/lib/meta/api/projectApis.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/nocodb/src/lib/meta/helpers/apiMetrics.ts b/packages/nocodb/src/lib/meta/helpers/apiMetrics.ts index 82c6a0c32b..e276c99ef9 100644 --- a/packages/nocodb/src/lib/meta/helpers/apiMetrics.ts +++ b/packages/nocodb/src/lib/meta/helpers/apiMetrics.ts @@ -1,5 +1,5 @@ import { Request } from 'express'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; const countMap = {}; @@ -9,7 +9,7 @@ const metrics = async (req: Request, c = 150) => { const event = `a:api:${req.route.path}:${req.method}`; countMap[event] = (countMap[event] || 0) + 1; if (countMap[event] >= c) { - Tele.event({ event }); + T.event({ event }); countMap[event] = 0; } }; diff --git a/packages/nocodb/src/lib/models/GalleryView.ts b/packages/nocodb/src/lib/models/GalleryView.ts index 141e37cbcc..8f530d2154 100644 --- a/packages/nocodb/src/lib/models/GalleryView.ts +++ b/packages/nocodb/src/lib/models/GalleryView.ts @@ -1,6 +1,6 @@ import Noco from '../Noco'; import { CacheGetType, CacheScope, MetaTable } from '../utils/globals'; -import { BoolType, GalleryColumnType, GalleryType, UITypes } from 'nocodb-sdk' +import { BoolType, GalleryColumnType, GalleryType, UITypes } from 'nocodb-sdk'; import View from './View'; import NocoCache from '../cache/NocoCache'; import { extractProps } from '../meta/helpers/extractProps'; diff --git a/packages/nocodb/src/lib/models/Hook.ts b/packages/nocodb/src/lib/models/Hook.ts index 47a640d8d9..1761dc8c12 100644 --- a/packages/nocodb/src/lib/models/Hook.ts +++ b/packages/nocodb/src/lib/models/Hook.ts @@ -1,42 +1,42 @@ -import { BoolType, HookType } from 'nocodb-sdk' +import { BoolType, HookReqType, HookType } from 'nocodb-sdk' import { CacheDelDirection, CacheGetType, CacheScope, MetaTable, -} from '../utils/globals'; -import Noco from '../Noco'; -import Model from './Model'; -import NocoCache from '../cache/NocoCache'; -import Filter from './Filter'; -import HookFilter from './HookFilter'; -import { extractProps } from '../meta/helpers/extractProps'; +} from '../utils/globals' +import Noco from '../Noco' +import Model from './Model' +import NocoCache from '../cache/NocoCache' +import Filter from './Filter' +import HookFilter from './HookFilter' +import { extractProps } from '../meta/helpers/extractProps' export default class Hook implements HookType { - id?: string; - fk_model_id?: string; - title?: string; - description?: string; - env?: string; - type?: string; - event?: 'after' | 'before'; - operation?: 'insert' | 'delete' | 'update'; - async?: BoolType; - payload?: string; - url?: string; - headers?: string; - condition?: BoolType; - notification?: string; - retries?: number; - retry_interval?: number; - timeout?: number; - active?: BoolType; + id?: string + fk_model_id?: string + title?: string + description?: string + env?: string + type?: string + event?: 'after' | 'before' + operation?: 'insert' | 'delete' | 'update' + async?: BoolType + payload?: string + url?: string + headers?: string + condition?: BoolType + notification?: string + retries?: number + retry_interval?: number + timeout?: number + active?: BoolType - project_id?: string; - base_id?: string; + project_id?: string + base_id?: string - constructor(hook: Partial) { - Object.assign(this, hook); + constructor(hook: Partial) { + Object.assign(this, hook) } public static async get(hookId: string, ncMeta = Noco.ncMeta) { @@ -44,17 +44,17 @@ export default class Hook implements HookType { hookId && (await NocoCache.get( `${CacheScope.HOOK}:${hookId}`, - CacheGetType.TYPE_OBJECT - )); + CacheGetType.TYPE_OBJECT, + )) if (!hook) { - hook = await ncMeta.metaGet2(null, null, MetaTable.HOOKS, hookId); - await NocoCache.set(`${CacheScope.HOOK}:${hookId}`, hook); + hook = await ncMeta.metaGet2(null, null, MetaTable.HOOKS, hookId) + await NocoCache.set(`${CacheScope.HOOK}:${hookId}`, hook) } - return hook && new Hook(hook); + return hook && new Hook(hook) } public async getFilters(ncMeta = Noco.ncMeta) { - return await Filter.rootFilterListByHook({ hookId: this.id }, ncMeta); + return await Filter.rootFilterListByHook({ hookId: this.id }, ncMeta) } // public static async insert(hook: Partial) { @@ -81,9 +81,9 @@ export default class Hook implements HookType { event?: 'after' | 'before'; operation?: 'insert' | 'delete' | 'update'; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { - let hooks = await NocoCache.getList(CacheScope.HOOK, [param.fk_model_id]); + let hooks = await NocoCache.getList(CacheScope.HOOK, [param.fk_model_id]) if (!hooks.length) { hooks = await ncMeta.metaList(null, null, MetaTable.HOOKS, { condition: { @@ -96,31 +96,29 @@ export default class Hook implements HookType { orderBy: { created_at: 'asc', }, - }); - await NocoCache.setList(CacheScope.HOOK, [param.fk_model_id], hooks); + }) + await NocoCache.setList(CacheScope.HOOK, [param.fk_model_id], hooks) } // filter event & operation if (param.event) { hooks = hooks.filter( - (h) => h.event?.toLowerCase() === param.event?.toLowerCase() - ); + (h) => h.event?.toLowerCase() === param.event?.toLowerCase(), + ) } if (param.operation) { hooks = hooks.filter( - (h) => h.operation?.toLowerCase() === param.operation?.toLowerCase() - ); + (h) => h.operation?.toLowerCase() === param.operation?.toLowerCase(), + ) } - return hooks?.map((h) => new Hook(h)); + return hooks?.map((h) => new Hook(h)) } public static async insert( - hook: Partial< - Hook & { - created_at?; - updated_at?; - } - >, - ncMeta = Noco.ncMeta + hook: Partial, + ncMeta = Noco.ncMeta, ) { const insertObj = extractProps(hook, [ 'fk_model_id', @@ -142,49 +140,49 @@ export default class Hook implements HookType { 'base_id', 'created_at', 'updated_at', - ]); + ]) if (insertObj.event) { - insertObj.event = insertObj.event.toLowerCase() as 'after' | 'before'; + insertObj.event = insertObj.event.toLowerCase() as 'after' | 'before' } if (insertObj.operation) { insertObj.operation = insertObj.operation.toLowerCase() as | 'insert' | 'delete' - | 'update'; + | 'update' } if (insertObj.notification && typeof insertObj.notification === 'object') { - insertObj.notification = JSON.stringify(insertObj.notification); + insertObj.notification = JSON.stringify(insertObj.notification) } if (!(hook.project_id && hook.base_id)) { - const model = await Model.getByIdOrName({ id: hook.fk_model_id }, ncMeta); - insertObj.project_id = model.project_id; - insertObj.base_id = model.base_id; + const model = await Model.getByIdOrName({ id: hook.fk_model_id }, ncMeta) + insertObj.project_id = model.project_id + insertObj.base_id = model.base_id } const { id } = await ncMeta.metaInsert2( null, null, MetaTable.HOOKS, - insertObj - ); + insertObj, + ) await NocoCache.appendToList( CacheScope.HOOK, [hook.fk_model_id], - `${CacheScope.HOOK}:${id}` - ); + `${CacheScope.HOOK}:${id}`, + ) - return this.get(id, ncMeta); + return this.get(id, ncMeta) } public static async update( hookId: string, hook: Partial, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { const updateObj = extractProps(hook, [ 'title', @@ -203,38 +201,38 @@ export default class Hook implements HookType { 'retry_interval', 'timeout', 'active', - ]); + ]) if (updateObj.event) { - updateObj.event = updateObj.event.toLowerCase() as 'after' | 'before'; + updateObj.event = updateObj.event.toLowerCase() as 'after' | 'before' } if (updateObj.operation) { updateObj.operation = updateObj.operation.toLowerCase() as | 'insert' | 'delete' - | 'update'; + | 'update' } if (updateObj.notification && typeof updateObj.notification === 'object') { - updateObj.notification = JSON.stringify(updateObj.notification); + updateObj.notification = JSON.stringify(updateObj.notification) } // get existing cache - const key = `${CacheScope.HOOK}:${hookId}`; - let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.HOOK}:${hookId}` + let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (o) { // update data - o = { ...o, ...updateObj }; + o = { ...o, ...updateObj } // replace notification - o.notification = updateObj.notification; + o.notification = updateObj.notification // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta - await ncMeta.metaUpdate(null, null, MetaTable.HOOKS, updateObj, hookId); + await ncMeta.metaUpdate(null, null, MetaTable.HOOKS, updateObj, hookId) - return this.get(hookId, ncMeta); + return this.get(hookId, ncMeta) } static async delete(hookId: any, ncMeta = Noco.ncMeta) { @@ -245,22 +243,22 @@ export default class Hook implements HookType { MetaTable.FILTER_EXP, { condition: { fk_hook_id: hookId }, - } - ); + }, + ) for (const filter of filterList) { await NocoCache.deepDel( CacheScope.FILTER_EXP, `${CacheScope.FILTER_EXP}:${filter.id}`, - CacheDelDirection.CHILD_TO_PARENT - ); - await HookFilter.delete(filter.id); + CacheDelDirection.CHILD_TO_PARENT, + ) + await HookFilter.delete(filter.id) } // Delete Hook await NocoCache.deepDel( CacheScope.HOOK, `${CacheScope.HOOK}:${hookId}`, - CacheDelDirection.CHILD_TO_PARENT - ); - return await ncMeta.metaDelete(null, null, MetaTable.HOOKS, hookId); + CacheDelDirection.CHILD_TO_PARENT, + ) + return await ncMeta.metaDelete(null, null, MetaTable.HOOKS, hookId) } } diff --git a/packages/nocodb/src/lib/models/KanbanView.ts b/packages/nocodb/src/lib/models/KanbanView.ts index 6ac1b65d09..9bff3fbed4 100644 --- a/packages/nocodb/src/lib/models/KanbanView.ts +++ b/packages/nocodb/src/lib/models/KanbanView.ts @@ -1,5 +1,5 @@ import Noco from '../Noco'; -import { KanbanType, UITypes } from 'nocodb-sdk'; +import { BoolType, KanbanType, UITypes } from 'nocodb-sdk'; import { CacheGetType, CacheScope, MetaTable } from '../utils/globals'; import View from './View'; import NocoCache from '../cache/NocoCache'; @@ -16,12 +16,12 @@ export default class KanbanView implements KanbanType { // below fields are not in use at this moment // keep them for time being - show?: boolean; + show?: BoolType; order?: number; uuid?: string; - public?: boolean; + public?: BoolType; password?: string; - show_all_fields?: boolean; + show_all_fields?: BoolType; constructor(data: KanbanView) { Object.assign(this, data); diff --git a/packages/nocodb/src/lib/models/Model.ts b/packages/nocodb/src/lib/models/Model.ts index eb32a3cc7b..4371c2841c 100644 --- a/packages/nocodb/src/lib/models/Model.ts +++ b/packages/nocodb/src/lib/models/Model.ts @@ -1,9 +1,9 @@ -import Noco from '../Noco' -import { parseMetaProp } from '../utils/modelUtils' -import Column from './Column' -import NocoCache from '../cache/NocoCache' -import { XKnex } from '../db/sql-data-mapper' -import { BaseModelSqlv2 } from '../db/sql-data-mapper/lib/sql/BaseModelSqlv2' +import Noco from '../Noco'; +import { parseMetaProp } from '../utils/modelUtils'; +import Column from './Column'; +import NocoCache from '../cache/NocoCache'; +import { XKnex } from '../db/sql-data-mapper'; +import { BaseModelSqlv2 } from '../db/sql-data-mapper/lib/sql/BaseModelSqlv2'; import { isVirtualCol, ModelTypes, @@ -12,52 +12,52 @@ import { TableType, UITypes, ViewTypes, -} from 'nocodb-sdk' +} from 'nocodb-sdk'; import { CacheDelDirection, CacheGetType, CacheScope, MetaTable, -} from '../utils/globals' -import View from './View' -import { NcError } from '../meta/helpers/catchError' -import Audit from './Audit' -import { sanitize } from '../db/sql-data-mapper/lib/sql/helpers/sanitize' -import { extractProps } from '../meta/helpers/extractProps' +} from '../utils/globals'; +import View from './View'; +import { NcError } from '../meta/helpers/catchError'; +import Audit from './Audit'; +import { sanitize } from '../db/sql-data-mapper/lib/sql/helpers/sanitize'; +import { extractProps } from '../meta/helpers/extractProps'; export default class Model implements TableType { - copy_enabled: BoolType - created_at: Date | number | string - base_id: 'db' | string - deleted: BoolType - enabled: BoolType - export_enabled: BoolType - id: string - order: number - parent_id: string - password: string - pin: BoolType - project_id: string - schema: any - show_all_fields: boolean - tags: string - type: ModelTypes - updated_at: Date | number | string - - table_name: string - title: string - - mm: BoolType - - uuid: string - - columns?: Column[] - columnsById?: { [id: string]: Column } - views?: View[] - meta?: Record | string + copy_enabled: BoolType; + created_at: Date | number | string; + base_id: 'db' | string; + deleted: BoolType; + enabled: BoolType; + export_enabled: BoolType; + id: string; + order: number; + parent_id: string; + password: string; + pin: BoolType; + project_id: string; + schema: any; + show_all_fields: boolean; + tags: string; + type: ModelTypes; + updated_at: Date | number | string; + + table_name: string; + title: string; + + mm: BoolType; + + uuid: string; + + columns?: Column[]; + columnsById?: { [id: string]: Column }; + views?: View[]; + meta?: Record | string; constructor(data: Partial) { - Object.assign(this, data) + Object.assign(this, data); } public async getColumns(ncMeta = Noco.ncMeta): Promise { @@ -65,34 +65,34 @@ export default class Model implements TableType { { fk_model_id: this.id, }, - ncMeta, - ) - return this.columns + ncMeta + ); + return this.columns; } // @ts-ignore public async getViews(force = false, ncMeta = Noco.ncMeta): Promise { - this.views = await View.listWithInfo(this.id, ncMeta) - return this.views + this.views = await View.listWithInfo(this.id, ncMeta); + return this.views; } public get primaryKey(): Column { - if (!this.columns) return null - return this.columns?.find((c) => c.pk) + if (!this.columns) return null; + return this.columns?.find((c) => c.pk); } public get primaryKeys(): Column[] { - if (!this.columns) return null - return this.columns?.filter((c) => c.pk) + if (!this.columns) return null; + return this.columns?.filter((c) => c.pk); } public get displayValue(): Column { - if (!this.columns) return null - const pCol = this.columns?.find((c) => c.pv) - if (pCol) return pCol - const pkIndex = this.columns.indexOf(this.primaryKey) - if (pkIndex < this.columns.length - 1) return this.columns[pkIndex + 1] - return this.columns[0] + if (!this.columns) return null; + const pCol = this.columns?.find((c) => c.pv); + if (pCol) return pCol; + const pkIndex = this.columns.indexOf(this.primaryKey); + if (pkIndex < this.columns.length - 1) return this.columns[pkIndex + 1]; + return this.columns[0]; } public static async insert( @@ -104,7 +104,7 @@ export default class Model implements TableType { updated_at?: any; type?: ModelTypes; }, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ) { const insertObj = extractProps(model, [ 'table_name', @@ -115,9 +115,9 @@ export default class Model implements TableType { 'created_at', 'updated_at', 'id', - ]) + ]); - insertObj.mm = !!insertObj.mm + insertObj.mm = !!insertObj.mm; if (!insertObj.order) { insertObj.order = await ncMeta.metaGetNextOrder( @@ -125,26 +125,26 @@ export default class Model implements TableType { { project_id: projectId, base_id: baseId, - }, - ) + } + ); } if (!insertObj.type) { - insertObj.type = ModelTypes.TABLE + insertObj.type = ModelTypes.TABLE; } const { id } = await ncMeta.metaInsert2( projectId, baseId, MetaTable.MODELS, - insertObj, - ) + insertObj + ); await NocoCache.appendToList( CacheScope.MODEL, [projectId], - `${CacheScope.MODEL}:${id}`, - ) + `${CacheScope.MODEL}:${id}` + ); const view = await View.insert( { @@ -155,14 +155,14 @@ export default class Model implements TableType { created_at: model.created_at, updated_at: model.updated_at, }, - ncMeta, - ) + ncMeta + ); for (const column of model?.columns || []) { - await Column.insert({ ...column, fk_model_id: id, view } as any, ncMeta) + await Column.insert({ ...column, fk_model_id: id, view } as any, ncMeta); } - return this.getWithInfo({ id }, ncMeta) + return this.getWithInfo({ id }, ncMeta); } public static async list( @@ -173,13 +173,13 @@ export default class Model implements TableType { project_id: string; base_id: string; }, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ): Promise { - let modelList = [] + let modelList = []; if (base_id) { - await NocoCache.getList(CacheScope.MODEL, [project_id, base_id]) + await NocoCache.getList(CacheScope.MODEL, [project_id, base_id]); } else { - await NocoCache.getList(CacheScope.MODEL, [project_id]) + await NocoCache.getList(CacheScope.MODEL, [project_id]); } if (!modelList.length) { modelList = await ncMeta.metaList2( @@ -190,30 +190,30 @@ export default class Model implements TableType { orderBy: { order: 'asc', }, - }, - ) + } + ); // parse meta of each model for (const model of modelList) { - model.meta = parseMetaProp(model) + model.meta = parseMetaProp(model); } if (base_id) { await NocoCache.setList( CacheScope.MODEL, [project_id, base_id], - modelList, - ) + modelList + ); } else { - await NocoCache.setList(CacheScope.MODEL, [project_id], modelList) + await NocoCache.setList(CacheScope.MODEL, [project_id], modelList); } } modelList.sort( (a, b) => (a.order != null ? a.order : Infinity) - - (b.order != null ? b.order : Infinity), - ) - return modelList.map((m) => new Model(m)) + (b.order != null ? b.order : Infinity) + ); + return modelList.map((m) => new Model(m)); } public static async listWithInfo( @@ -224,33 +224,33 @@ export default class Model implements TableType { project_id: string; db_alias: string; }, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ): Promise { let modelList = await NocoCache.getList(CacheScope.MODEL, [ project_id, db_alias, - ]) + ]); if (!modelList.length) { modelList = await ncMeta.metaList2( project_id, db_alias, - MetaTable.MODELS, - ) + MetaTable.MODELS + ); // parse meta of each model for (const model of modelList) { - model.meta = parseMetaProp(model) + model.meta = parseMetaProp(model); } - await NocoCache.setList(CacheScope.MODEL, [project_id], modelList) + await NocoCache.setList(CacheScope.MODEL, [project_id], modelList); } - return modelList.map((m) => new Model(m)) + return modelList.map((m) => new Model(m)); } public static async clear({ id }: { id: string }): Promise { - await NocoCache.delAll(CacheScope.MODEL, `*${id}*`) - await Column.clearList({ fk_model_id: id }) + await NocoCache.delAll(CacheScope.MODEL, `*${id}*`); + await Column.clearList({ fk_model_id: id }); } public static async get(id: string, ncMeta = Noco.ncMeta): Promise { @@ -258,47 +258,47 @@ export default class Model implements TableType { id && (await NocoCache.get( `${CacheScope.MODEL}:${id}`, - CacheGetType.TYPE_OBJECT, - )) + CacheGetType.TYPE_OBJECT + )); if (!modelData) { - modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, id) + modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, id); if (modelData) { - modelData.meta = parseMetaProp(modelData) - await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData) + modelData.meta = parseMetaProp(modelData); + await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData); } } - return modelData && new Model(modelData) + return modelData && new Model(modelData); } public static async getByIdOrName( args: | { - project_id: string; - base_id: string; - table_name: string; - } + project_id: string; + base_id: string; + table_name: string; + } | { - id?: string; - }, - ncMeta = Noco.ncMeta, + id?: string; + }, + ncMeta = Noco.ncMeta ): Promise { - const k = 'id' in args ? args?.id : args + const k = 'id' in args ? args?.id : args; let modelData = k && (await NocoCache.get( `${CacheScope.MODEL}:${k}`, - CacheGetType.TYPE_OBJECT, - )) + CacheGetType.TYPE_OBJECT + )); if (!modelData) { - modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, k) - modelData.meta = parseMetaProp(modelData) + modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, k); + modelData.meta = parseMetaProp(modelData); } if (modelData) { - await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData) - return new Model(modelData) + await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData); + return new Model(modelData); } - return null + return null; } public static async getWithInfo( @@ -309,14 +309,14 @@ export default class Model implements TableType { table_name?: string; id?: string; }, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ): Promise { let modelData = id && (await NocoCache.get( `${CacheScope.MODEL}:${id}`, - CacheGetType.TYPE_OBJECT, - )) + CacheGetType.TYPE_OBJECT + )); if (!modelData) { modelData = await ncMeta.metaGet2( null, @@ -324,23 +324,23 @@ export default class Model implements TableType { MetaTable.MODELS, id || { table_name, - }, - ) - modelData.meta = parseMetaProp(modelData) - await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData) + } + ); + modelData.meta = parseMetaProp(modelData); + await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData); // modelData.filters = await Filter.getFilterObject({ // viewId: modelData.id // }); // modelData.sorts = await Sort.list({ modelId: modelData.id }); } if (modelData) { - const m = new Model(modelData) - const columns = await m.getColumns(ncMeta) - await m.getViews(false, ncMeta) - m.columnsById = columns.reduce((agg, c) => ({ ...agg, [c.id]: c }), {}) - return m + const m = new Model(modelData); + const columns = await m.getColumns(ncMeta); + await m.getViews(false, ncMeta); + m.columnsById = columns.reduce((agg, c) => ({ ...agg, [c.id]: c }), {}); + return m; } - return null + return null; } public static async getBaseModelSQL( @@ -350,60 +350,60 @@ export default class Model implements TableType { dbDriver: XKnex; model?: Model; }, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ): Promise { - const model = args?.model || (await this.get(args.id, ncMeta)) + const model = args?.model || (await this.get(args.id, ncMeta)); return new BaseModelSqlv2({ dbDriver: args.dbDriver, viewId: args.viewId, model, - }) + }); } async delete(ncMeta = Noco.ncMeta, force = false): Promise { - await Audit.deleteRowComments(this.id) + await Audit.deleteRowComments(this.id); for (const view of await this.getViews(true)) { - await view.delete() + await view.delete(); } for (const col of await this.getColumns(ncMeta)) { - let colOptionTableName = null - let cacheScopeName = null + let colOptionTableName = null; + let cacheScopeName = null; switch (col.uidt) { case UITypes.Rollup: - colOptionTableName = MetaTable.COL_ROLLUP - cacheScopeName = CacheScope.COL_ROLLUP - break + colOptionTableName = MetaTable.COL_ROLLUP; + cacheScopeName = CacheScope.COL_ROLLUP; + break; case UITypes.Lookup: - colOptionTableName = MetaTable.COL_LOOKUP - cacheScopeName = CacheScope.COL_LOOKUP - break + colOptionTableName = MetaTable.COL_LOOKUP; + cacheScopeName = CacheScope.COL_LOOKUP; + break; case UITypes.ForeignKey: case UITypes.LinkToAnotherRecord: - colOptionTableName = MetaTable.COL_RELATIONS - cacheScopeName = CacheScope.COL_RELATION - break + colOptionTableName = MetaTable.COL_RELATIONS; + cacheScopeName = CacheScope.COL_RELATION; + break; case UITypes.MultiSelect: case UITypes.SingleSelect: - colOptionTableName = MetaTable.COL_SELECT_OPTIONS - cacheScopeName = CacheScope.COL_SELECT_OPTION - break + colOptionTableName = MetaTable.COL_SELECT_OPTIONS; + cacheScopeName = CacheScope.COL_SELECT_OPTION; + break; case UITypes.Formula: - colOptionTableName = MetaTable.COL_FORMULA - cacheScopeName = CacheScope.COL_FORMULA - break + colOptionTableName = MetaTable.COL_FORMULA; + cacheScopeName = CacheScope.COL_FORMULA; + break; } if (colOptionTableName && cacheScopeName) { await ncMeta.metaDelete(null, null, colOptionTableName, { fk_column_id: col.id, - }) + }); await NocoCache.deepDel( cacheScopeName, `${cacheScopeName}:${col.id}`, - CacheDelDirection.CHILD_TO_PARENT, - ) + CacheDelDirection.CHILD_TO_PARENT + ); } } @@ -416,82 +416,82 @@ export default class Model implements TableType { condition: { fk_related_model_id: this.id, }, - }, - ) + } + ); for (const col of leftOverColumns) { await NocoCache.deepDel( CacheScope.COL_RELATION, `${CacheScope.COL_RELATION}:${col.fk_column_id}`, - CacheDelDirection.CHILD_TO_PARENT, - ) + CacheDelDirection.CHILD_TO_PARENT + ); } await ncMeta.metaDelete(null, null, MetaTable.COL_RELATIONS, { fk_related_model_id: this.id, - }) + }); } await NocoCache.deepDel( CacheScope.COLUMN, `${CacheScope.COLUMN}:${this.id}`, - CacheDelDirection.CHILD_TO_PARENT, - ) + CacheDelDirection.CHILD_TO_PARENT + ); await ncMeta.metaDelete(null, null, MetaTable.COLUMNS, { fk_model_id: this.id, - }) + }); await NocoCache.deepDel( CacheScope.MODEL, `${CacheScope.MODEL}:${this.id}`, - CacheDelDirection.CHILD_TO_PARENT, - ) - await ncMeta.metaDelete(null, null, MetaTable.MODELS, this.id) + CacheDelDirection.CHILD_TO_PARENT + ); + await ncMeta.metaDelete(null, null, MetaTable.MODELS, this.id); - await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.id}`) - await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.title}`) - return true + await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.id}`); + await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.title}`); + return true; } async mapAliasToColumn(data) { - const insertObj = {} + const insertObj = {}; for (const col of await this.getColumns()) { - if (isVirtualCol(col)) continue + if (isVirtualCol(col)) continue; let val = data?.[col.column_name] !== undefined ? data?.[col.column_name] - : data?.[col.title] + : data?.[col.title]; if (val !== undefined) { if (col.uidt === UITypes.Attachment && typeof val !== 'string') { - val = JSON.stringify(val) + val = JSON.stringify(val); } - insertObj[sanitize(col.column_name)] = val + insertObj[sanitize(col.column_name)] = val; } } - return insertObj + return insertObj; } static async updateAliasAndTableName( tableId, title: string, table_name: string, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ) { if (!title) { - NcError.badRequest('Missing \'title\' property in body') + NcError.badRequest("Missing 'title' property in body"); } if (!table_name) { - NcError.badRequest('Missing \'table_name\' property in body') + NcError.badRequest("Missing 'table_name' property in body"); } // get existing cache - const key = `${CacheScope.MODEL}:${tableId}` - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) + const key = `${CacheScope.MODEL}:${tableId}`; + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); // update alias if (o) { - o.title = title - o.table_name = table_name + o.title = title; + o.table_name = table_name; // set cache - await NocoCache.set(key, o) + await NocoCache.set(key, o); } // set meta return await ncMeta.metaUpdate( @@ -502,19 +502,19 @@ export default class Model implements TableType { title, table_name, }, - tableId, - ) + tableId + ); } static async markAsMmTable(tableId, isMm = true, ncMeta = Noco.ncMeta) { // get existing cache - const key = `${CacheScope.MODEL}:${tableId}` - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) + const key = `${CacheScope.MODEL}:${tableId}`; + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); // update alias if (o) { - o.mm = isMm + o.mm = isMm; // set cache - await NocoCache.set(key, o) + await NocoCache.set(key, o); } // set meta return await ncMeta.metaUpdate( @@ -524,40 +524,40 @@ export default class Model implements TableType { { mm: isMm, }, - tableId, - ) + tableId + ); } async getAliasColMapping() { return (await this.getColumns()).reduce((o, c) => { if (c.column_name) { - o[c.title] = c.column_name + o[c.title] = c.column_name; } - return o - }, {}) + return o; + }, {}); } async getColAliasMapping() { return (await this.getColumns()).reduce((o, c) => { if (c.column_name) { - o[c.column_name] = c.title + o[c.column_name] = c.title; } - return o - }, {}) + return o; + }, {}); } static async updateOrder( tableId: string, order: number, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ) { // get existing cache - const key = `${CacheScope.MODEL}:${tableId}` - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) + const key = `${CacheScope.MODEL}:${tableId}`; + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); if (o) { - o.order = order + o.order = order; // set cache - await NocoCache.set(key, o) + await NocoCache.set(key, o); } // set meta return await ncMeta.metaUpdate( @@ -567,29 +567,29 @@ export default class Model implements TableType { { order, }, - tableId, - ) + tableId + ); } static async updatePrimaryColumn( tableId: string, columnId: string, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ) { - const model = await this.getWithInfo({ id: tableId }) - const newPvCol = model.columns.find((c) => c.id === columnId) + const model = await this.getWithInfo({ id: tableId }); + const newPvCol = model.columns.find((c) => c.id === columnId); - if (!newPvCol) NcError.badRequest('Column not found') + if (!newPvCol) NcError.badRequest('Column not found'); // drop existing primary column/s for (const col of model.columns?.filter((c) => c.pv) || []) { // get existing cache - const key = `${CacheScope.COLUMN}:${col.id}` - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) + const key = `${CacheScope.COLUMN}:${col.id}`; + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); if (o) { - o.pv = false + o.pv = false; // set cache - await NocoCache.set(key, o) + await NocoCache.set(key, o); } // set meta await ncMeta.metaUpdate( @@ -599,17 +599,17 @@ export default class Model implements TableType { { pv: false, }, - col.id, - ) + col.id + ); } // get existing cache - const key = `${CacheScope.COLUMN}:${newPvCol.id}` - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) + const key = `${CacheScope.COLUMN}:${newPvCol.id}`; + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); if (o) { - o.pv = true + o.pv = true; // set cache - await NocoCache.set(key, o) + await NocoCache.set(key, o); } // set meta await ncMeta.metaUpdate( @@ -619,8 +619,8 @@ export default class Model implements TableType { { pv: true, }, - newPvCol.id, - ) + newPvCol.id + ); const grid_views_with_column = await ncMeta.metaList2( null, @@ -630,26 +630,26 @@ export default class Model implements TableType { condition: { fk_column_id: newPvCol.id, }, - }, - ) + } + ); if (grid_views_with_column.length) { for (const gv of grid_views_with_column) { - await View.fixPVColumnForView(gv.fk_view_id, ncMeta) + await View.fixPVColumnForView(gv.fk_view_id, ncMeta); } } - return true + return true; } static async setAsMm(id: any, ncMeta = Noco.ncMeta) { // get existing cache - const key = `${CacheScope.MODEL}:${id}` - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) + const key = `${CacheScope.MODEL}:${id}`; + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); if (o) { - o.mm = true + o.mm = true; // set cache - await NocoCache.set(key, o) + await NocoCache.set(key, o); } // set meta await ncMeta.metaUpdate( @@ -659,8 +659,8 @@ export default class Model implements TableType { { mm: true, }, - id, - ) + id + ); } static async getByAliasOrId( @@ -673,69 +673,69 @@ export default class Model implements TableType { base_id?: string; aliasOrId: string; }, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ) { const modelId = project_id && aliasOrId && (await NocoCache.get( `${CacheScope.MODEL}:${project_id}:${aliasOrId}`, - CacheGetType.TYPE_OBJECT, - )) + CacheGetType.TYPE_OBJECT + )); if (!modelId) { const model = base_id ? await ncMeta.metaGet2( - null, - null, - MetaTable.MODELS, - { project_id, base_id }, - null, - { - _or: [ - { - id: { - eq: aliasOrId, + null, + null, + MetaTable.MODELS, + { project_id, base_id }, + null, + { + _or: [ + { + id: { + eq: aliasOrId, + }, }, - }, - { - title: { - eq: aliasOrId, + { + title: { + eq: aliasOrId, + }, }, - }, - ], - }, - ) + ], + } + ) : await ncMeta.metaGet2( - null, - null, - MetaTable.MODELS, - { project_id }, - null, - { - _or: [ - { - id: { - eq: aliasOrId, + null, + null, + MetaTable.MODELS, + { project_id }, + null, + { + _or: [ + { + id: { + eq: aliasOrId, + }, }, - }, - { - title: { - eq: aliasOrId, + { + title: { + eq: aliasOrId, + }, }, - }, - ], - }, - ) + ], + } + ); if (model) { await NocoCache.set( `${CacheScope.MODEL}:${project_id}:${aliasOrId}`, - model.id, - ) - await NocoCache.set(`${CacheScope.MODEL}:${model.id}`, model) + model.id + ); + await NocoCache.set(`${CacheScope.MODEL}:${model.id}`, model); } - return model && new Model(model) + return model && new Model(model); } - return modelId && this.get(modelId) + return modelId && this.get(modelId); } static async checkTitleAvailable( @@ -745,7 +745,7 @@ export default class Model implements TableType { base_id, exclude_id, }: { table_name; project_id; base_id; exclude_id? }, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ) { return !(await ncMeta.metaGet2( project_id, @@ -755,8 +755,8 @@ export default class Model implements TableType { table_name, }, null, - exclude_id && { id: { neq: exclude_id } }, - )) + exclude_id && { id: { neq: exclude_id } } + )); } static async checkAliasAvailable( @@ -766,7 +766,7 @@ export default class Model implements TableType { base_id, exclude_id, }: { title; project_id; base_id; exclude_id? }, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ) { return !(await ncMeta.metaGet2( project_id, @@ -776,33 +776,32 @@ export default class Model implements TableType { title, }, null, - exclude_id && { id: { neq: exclude_id } }, - )) + exclude_id && { id: { neq: exclude_id } } + )); } async getAliasColObjMap() { return (await this.getColumns()).reduce( (sortAgg, c) => ({ ...sortAgg, [c.title]: c }), - {}, - ) + {} + ); } // For updating table meta static async updateMeta( tableId: string, meta: string | Record, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ) { // get existing cache - const key = `${CacheScope.MODEL}:${tableId}` - const existingCache = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) + const key = `${CacheScope.MODEL}:${tableId}`; + const existingCache = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); if (existingCache) { try { - existingCache.meta = typeof meta === 'string' ? JSON.parse(meta) : meta + existingCache.meta = typeof meta === 'string' ? JSON.parse(meta) : meta; // set cache - await NocoCache.set(key, existingCache) - } catch { - } + await NocoCache.set(key, existingCache); + } catch {} } // set meta return await ncMeta.metaUpdate( @@ -812,7 +811,7 @@ export default class Model implements TableType { { meta: typeof meta === 'object' ? JSON.stringify(meta) : meta, }, - tableId, - ) + tableId + ); } } diff --git a/packages/nocodb/src/lib/models/Project.ts b/packages/nocodb/src/lib/models/Project.ts index a3774b96d9..5ee799d1ab 100644 --- a/packages/nocodb/src/lib/models/Project.ts +++ b/packages/nocodb/src/lib/models/Project.ts @@ -1,6 +1,6 @@ import Base from './/Base'; import Noco from '../Noco'; -import { BoolType, MetaType, ProjectType } from 'nocodb-sdk' +import { BoolType, MetaType, ProjectType } from 'nocodb-sdk'; import { CacheDelDirection, CacheGetType, diff --git a/packages/nocodb/src/lib/services/apiTokenService.ts b/packages/nocodb/src/lib/services/apiTokenService.ts index aa106328f6..af94b7936b 100644 --- a/packages/nocodb/src/lib/services/apiTokenService.ts +++ b/packages/nocodb/src/lib/services/apiTokenService.ts @@ -1,5 +1,5 @@ import { ApiTokenReqType, OrgUserRoles } from 'nocodb-sdk'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { NcError } from '../meta/helpers/catchError'; import ApiToken from '../models/ApiToken'; import User from '../models/User'; @@ -11,7 +11,7 @@ export async function apiTokenCreate(param: { userId: string; tokenBody: ApiTokenReqType; }) { - Tele.emit('evt', { evt_type: 'apiToken:created' }); + T.emit('evt', { evt_type: 'apiToken:created' }); return ApiToken.insert({ ...param.tokenBody, fk_user_id: param.userId }); } @@ -23,7 +23,7 @@ export async function apiTokenDelete(param: { token; user: User }) { ) { NcError.notFound('Token not found'); } - Tele.emit('evt', { evt_type: 'apiToken:deleted' }); + T.emit('evt', { evt_type: 'apiToken:deleted' }); // todo: verify token belongs to the user return await ApiToken.delete(param.token); diff --git a/packages/nocodb/src/lib/services/attachmentService.ts b/packages/nocodb/src/lib/services/attachmentService.ts index 7b2b6d3169..f4a9bfdca1 100644 --- a/packages/nocodb/src/lib/services/attachmentService.ts +++ b/packages/nocodb/src/lib/services/attachmentService.ts @@ -4,7 +4,7 @@ import { nanoid } from 'nanoid'; import path from 'path'; import slash from 'slash'; import mimetypes, { mimeIcons } from '../utils/mimeTypes'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; import Local from '../v1-legacy/plugins/adapters/storage/Local'; @@ -47,7 +47,7 @@ export async function upload(param: { }) ); - Tele.emit('evt', { evt_type: 'image:uploaded' }); + T.emit('evt', { evt_type: 'image:uploaded' }); return attachments; } @@ -97,7 +97,7 @@ export async function uploadViaURL(param: { }) ); - Tele.emit('evt', { evt_type: 'image:uploaded' }); + T.emit('evt', { evt_type: 'image:uploaded' }); return attachments; } diff --git a/packages/nocodb/src/lib/services/baseService.ts b/packages/nocodb/src/lib/services/baseService.ts index 1f390198c0..dbb5001bf0 100644 --- a/packages/nocodb/src/lib/services/baseService.ts +++ b/packages/nocodb/src/lib/services/baseService.ts @@ -2,7 +2,7 @@ import Project from '../models/Project'; import { BaseReqType } from 'nocodb-sdk'; import { syncBaseMigration } from '../meta/helpers/syncMigration'; import Base from '../models/Base'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { populateMeta } from '../meta/api/helpers'; export async function baseGetWithConfig(param: { baseId: any }) { @@ -29,7 +29,7 @@ export async function baseUpdate(param: { delete base.config; - Tele.emit('evt', { + T.emit('evt', { evt_type: 'base:updated', }); @@ -45,7 +45,7 @@ export async function baseList(param: { projectId: string }) { export async function baseDelete(param: { baseId: string }) { const base = await Base.get(param.baseId); await base.delete(); - Tele.emit('evt', { evt_type: 'base:deleted' }); + T.emit('evt', { evt_type: 'base:deleted' }); return true; } @@ -66,11 +66,11 @@ export async function baseCreate(param: { const info = await populateMeta(base, project); - Tele.emit('evt_api_created', info); + T.emit('evt_api_created', info); delete base.config; - Tele.emit('evt', { + T.emit('evt', { evt_type: 'base:created', }); diff --git a/packages/nocodb/src/lib/services/cacheService.ts b/packages/nocodb/src/lib/services/cacheService.ts index 6e3b83df86..968da7c33f 100644 --- a/packages/nocodb/src/lib/services/cacheService.ts +++ b/packages/nocodb/src/lib/services/cacheService.ts @@ -1,10 +1,10 @@ import NocoCache from '../cache/NocoCache'; export async function cacheGet() { - return await NocoCache.export(); + return await NocoCache.export(); } export async function cacheDelete() { - await NocoCache.destroy() - return true + await NocoCache.destroy(); + return true; } diff --git a/packages/nocodb/src/lib/services/columnService.ts b/packages/nocodb/src/lib/services/columnService.ts index c8c1f80f4e..f2c741f546 100644 --- a/packages/nocodb/src/lib/services/columnService.ts +++ b/packages/nocodb/src/lib/services/columnService.ts @@ -6,20 +6,21 @@ import { LinkToAnotherColumnReqType, LinkToAnotherRecordType, RelationTypes, - substituteColumnAliasWithIdInFormula, substituteColumnIdWithAliasInFormula, + substituteColumnAliasWithIdInFormula, + substituteColumnIdWithAliasInFormula, UITypes, -} from 'nocodb-sdk' +} from 'nocodb-sdk'; import formulaQueryBuilderv2 from '../db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2'; import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; import SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2'; -import { Altered } from '../meta/api/columnApis'; import { createHmAndBtColumn, generateFkName, randomID, - validateLookupPayload, validateRequiredField, + validateLookupPayload, + validateRequiredField, validateRollupPayload, -} from '../meta/api/helpers' +} from '../meta/api/helpers'; import { NcError } from '../meta/helpers/catchError'; import getColumnPropsFromUIDT from '../meta/helpers/getColumnPropsFromUIDT'; import { @@ -32,7 +33,7 @@ import NcMetaIO from '../meta/NcMetaIO'; import Audit from '../models/Audit'; import Base from '../models/Base'; import Column from '../models/Column'; -import FormulaColumn from '../models/FormulaColumn' +import FormulaColumn from '../models/FormulaColumn'; import KanbanView from '../models/KanbanView'; import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; import Model from '../models/Model'; @@ -40,14 +41,21 @@ import Project from '../models/Project'; import Noco from '../Noco'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; -import { Tele } from 'nc-help'; -import { MetaTable } from '../utils/globals' +import { T } from 'nc-help'; +import { MetaTable } from '../utils/globals'; + +export enum Altered { + NEW_COLUMN = 1, + DELETE_COLUMN = 4, + UPDATE_COLUMN = 8, +} export async function columnUpdate(param: { columnId: string; - column: ColumnReqType & {colOptions?: any}, - cookie?: any, + column: ColumnReqType & { colOptions?: any }; + cookie?: any; }) { + const { cookie } = param; const column = await Column.get({ colId: param.columnId }); const table = await Model.getWithInfo({ @@ -88,7 +96,10 @@ export async function columnUpdate(param: { NcError.badRequest('Duplicate column alias'); } - let colBody = { ...param.column } as Column & { formula?: string; formula_raw?: string }; + let colBody = { ...param.column } as Column & { + formula?: string; + formula_raw?: string; + }; if ( [ UITypes.Lookup, @@ -606,7 +617,7 @@ export async function columnUpdate(param: { await baseModel.bulkUpdateAll( { where: `(${column.title},eq,${ch.temp_title})` }, { [column.column_name]: newOp.title }, - { cookie: req } + { cookie } ); } } else if (column.uidt === UITypes.MultiSelect) { @@ -806,9 +817,9 @@ export async function columnUpdate(param: { }).then(() => {}); await table.getColumns(); - Tele.emit('evt', { evt_type: 'column:updated' }); + T.emit('evt', { evt_type: 'column:updated' }); - return table + return table; } export async function columnGet(param: { columnId: string }) { @@ -893,7 +904,7 @@ export async function columnAdd(param: { case UITypes.LinkToAnotherRecord: await createLTARColumn({ ...param, base, project }); - Tele.emit('evt', { evt_type: 'relation:created' }); + T.emit('evt', { evt_type: 'relation:created' }); break; case UITypes.QrCode: @@ -1098,10 +1109,9 @@ export async function columnAdd(param: { // ip: (req as any).clientIp, }).then(() => {}); - Tele.emit('evt', { evt_type: 'column:created' }); - - return table + T.emit('evt', { evt_type: 'column:created' }); + return table; } export async function columnDelete(param: { columnId: string }) { @@ -1255,7 +1265,7 @@ export async function columnDelete(param: { columnId: string }) { break; } } - Tele.emit('evt', { evt_type: 'raltion:deleted' }); + T.emit('evt', { evt_type: 'raltion:deleted' }); break; case UITypes.ForeignKey: { NcError.notImplemented(); @@ -1321,7 +1331,7 @@ export async function columnDelete(param: { columnId: string }) { ); } - Tele.emit('evt', { evt_type: 'column:deleted' }); + T.emit('evt', { evt_type: 'column:deleted' }); return table; } diff --git a/packages/nocodb/src/lib/services/dataService/export.ts b/packages/nocodb/src/lib/services/dataService/export.ts index 2ebc209262..f74e5299f9 100644 --- a/packages/nocodb/src/lib/services/dataService/export.ts +++ b/packages/nocodb/src/lib/services/dataService/export.ts @@ -1,25 +1,35 @@ import { Request, Response, Router } from 'express'; -import { isSystemColumn } from 'nocodb-sdk' +import { isSystemColumn } from 'nocodb-sdk'; import * as XLSX from 'xlsx'; -import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst' +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import { extractXlsxData, serializeCellValue } from '../../meta/api/dataApis/helpers' import { - extractCsvData, getViewAndModelByAliasOrId, - getViewAndModelFromRequestByAliasOrId, PathParams, -} from './helpers' + extractXlsxData, + serializeCellValue, +} from '../../meta/api/dataApis/helpers'; +import { + extractCsvData, + getViewAndModelByAliasOrId, + getViewAndModelFromRequestByAliasOrId, + PathParams, +} from './helpers'; import apiMetrics from '../../helpers/apiMetrics'; import View from '../../../models/View'; -async function excelDataExport(param:PathParams&{ - query: any; -}) { +async function excelDataExport( + param: PathParams & { + query: any; + } +) { const { model, view } = await getViewAndModelByAliasOrId(param); let targetView = view; if (!targetView) { targetView = await View.getDefaultView(model.id); } - const { offset, elapsed, data } = await extractXlsxData({view: targetView, query:req.query }); + const { offset, elapsed, data } = await extractXlsxData({ + view: targetView, + query: req.query, + }); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, data, targetView.title); const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); diff --git a/packages/nocodb/src/lib/services/dataService/helpers.ts b/packages/nocodb/src/lib/services/dataService/helpers.ts index 02c56c7a39..9654fc3dfc 100644 --- a/packages/nocodb/src/lib/services/dataService/helpers.ts +++ b/packages/nocodb/src/lib/services/dataService/helpers.ts @@ -1,18 +1,18 @@ -import { Request } from 'express' -import { nocoExecute } from 'nc-help' -import { isSystemColumn, UITypes } from 'nocodb-sdk' -import * as XLSX from 'xlsx' -import { BaseModelSqlv2 } from '../../db/sql-data-mapper/lib/sql/BaseModelSqlv2' -import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst' -import { NcError } from '../../meta/helpers/catchError' -import { Model, View } from '../../models' -import Base from '../../models/Base' -import Column from '../../models/Column' -import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn' -import LookupColumn from '../../models/LookupColumn' -import Project from '../../models/Project' -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2' - +import { Request } from 'express'; +import { nocoExecute } from 'nc-help'; +import { isSystemColumn, UITypes } from 'nocodb-sdk'; +import * as XLSX from 'xlsx'; +import { BaseModelSqlv2 } from '../../db/sql-data-mapper/lib/sql/BaseModelSqlv2'; +import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import { NcError } from '../../meta/helpers/catchError'; +import { Model, View } from '../../models'; +import Base from '../../models/Base'; +import Column from '../../models/Column'; +import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; +import LookupColumn from '../../models/LookupColumn'; +import Project from '../../models/Project'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import papaparse from 'papaparse'; export interface PathParams { projectName: string; @@ -41,8 +41,6 @@ export async function getViewAndModelByAliasOrId(param: { return { model, view }; } - - export async function extractXlsxData(view: View, req: Request) { const base = await Base.get(view.base_id); @@ -63,7 +61,12 @@ export async function extractXlsxData(view: View, req: Request) { dbDriver: NcConnectionMgrv2.get(base), }); - const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req); + const { offset, dbRows, elapsed } = await getDbRows({ + baseModel, + view, + siteUrl: (req as any).ncSiteUrl, + query: req.query, + }); const fields = req.query.fields as string[]; @@ -93,7 +96,12 @@ export async function extractCsvData(view: View, req: Request) { dbDriver: NcConnectionMgrv2.get(base), }); - const { offset, dbRows, elapsed } = await getDbRows(baseModel, view, req); + const { offset, dbRows, elapsed } = await getDbRows({ + baseModel, + view, + query: req.query, + siteUrl: (req as any).ncSiteUrl, + }); const data = papaparse.unparse( { @@ -118,70 +126,11 @@ export async function extractCsvData(view: View, req: Request) { return { offset, dbRows, elapsed, data }; } - -async function getDbRows(baseModel, view: View, req: Request) { - let offset = +req.query.offset || 0; - const limit = 100; - // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; - const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; - const dbRows = []; - const startTime = process.hrtime(); - let elapsed, temp; - - const listArgs: any = { ...req.query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - - for ( - elapsed = 0; - elapsed < timeout; - offset += limit, - temp = process.hrtime(startTime), - elapsed = temp[0] * 1000 + temp[1] / 1000000 - ) { - const rows = await nocoExecute( - await getAst({ - query: req.query, - includePkByDefault: false, - model: view.model, - view, - }), - await baseModel.list({ ...listArgs, offset, limit }), - {}, - req.query - ); - - if (!rows?.length) { - offset = -1; - break; - } - - for (const row of rows) { - const dbRow = { ...row }; - - for (const column of view.model.columns) { - if (isSystemColumn(column) && !view.show_system_fields) continue; - dbRow[column.title] = await serializeCellValue({ - value: row[column.title], - column, - siteUrl: req['ncSiteUrl'], - }); - } - dbRows.push(dbRow); - } - } - return { offset, dbRows, elapsed }; -} - export async function serializeCellValue({ - value, - column, - siteUrl, - }: { + value, + column, + siteUrl, +}: { column?: Column; value: any; siteUrl: string; @@ -209,34 +158,34 @@ export async function serializeCellValue({ ); } case UITypes.Lookup: - { - const colOptions = await column.getColOptions(); - const lookupColumn = await colOptions.getLookupColumn(); - return ( - await Promise.all( - [...(Array.isArray(value) ? value : [value])].map(async (v) => - serializeCellValue({ - value: v, - column: lookupColumn, - siteUrl, - }) + { + const colOptions = await column.getColOptions(); + const lookupColumn = await colOptions.getLookupColumn(); + return ( + await Promise.all( + [...(Array.isArray(value) ? value : [value])].map(async (v) => + serializeCellValue({ + value: v, + column: lookupColumn, + siteUrl, + }) + ) ) - ) - ).join(', '); - } + ).join(', '); + } break; case UITypes.LinkToAnotherRecord: - { - const colOptions = - await column.getColOptions(); - const relatedModel = await colOptions.getRelatedTable(); - await relatedModel.getColumns(); - return [...(Array.isArray(value) ? value : [value])] - .map((v) => { - return v[relatedModel.displayValue?.title]; - }) - .join(', '); - } + { + const colOptions = + await column.getColOptions(); + const relatedModel = await colOptions.getRelatedTable(); + await relatedModel.getColumns(); + return [...(Array.isArray(value) ? value : [value])] + .map((v) => { + return v[relatedModel.displayValue?.title]; + }) + .join(', '); + } break; default: if (value && typeof value === 'object') { @@ -263,9 +212,12 @@ export async function getColumnByIdOrName( return column; } - - -async function getDbRows(param: { baseModel:BaseModelSqlv2, view: View, query: any; siteUrl: string; }) { +export async function getDbRows(param: { + baseModel: BaseModelSqlv2; + view: View; + query: any; + siteUrl: string; +}) { const { baseModel, view, query = {}, siteUrl } = param; let offset = +query.offset || 0; const limit = 100; diff --git a/packages/nocodb/src/lib/services/dataService/index.ts b/packages/nocodb/src/lib/services/dataService/index.ts index 668da2a598..44e4b7bedb 100644 --- a/packages/nocodb/src/lib/services/dataService/index.ts +++ b/packages/nocodb/src/lib/services/dataService/index.ts @@ -4,7 +4,7 @@ import { NcError } from '../../meta/helpers/catchError'; import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; import { Base, Model, View } from '../../models'; import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; -import { getViewAndModelByAliasOrId, PathParams } from './helpers' +import { getViewAndModelByAliasOrId, PathParams } from './helpers'; export async function dataList(param: PathParams & { query: any }) { const { model, view } = await getViewAndModelByAliasOrId(param); @@ -322,7 +322,6 @@ export async function getGroupedDataList(param: { return data; } - export async function dataListByViewId(param: { viewId: string; query: any }) { const view = await View.get(param.viewId); @@ -795,3 +794,5 @@ export async function relationDataAdd(param: { return true; } + +export * from './helpers' diff --git a/packages/nocodb/src/lib/services/filterService.ts b/packages/nocodb/src/lib/services/filterService.ts index 8b40b4b813..c52089118b 100644 --- a/packages/nocodb/src/lib/services/filterService.ts +++ b/packages/nocodb/src/lib/services/filterService.ts @@ -1,68 +1,67 @@ -import { FilterReqType } from 'nocodb-sdk' -import Filter from '../models/Filter' -import { Tele } from 'nc-help'; +import { FilterReqType } from 'nocodb-sdk'; +import Filter from '../models/Filter'; +import { T } from 'nc-help'; -export async function hookFilterCreate(param: { filter: FilterReqType; hookId: any }) { +export async function hookFilterCreate(param: { + filter: FilterReqType; + hookId: any; +}) { const filter = await Filter.insert({ ...param.filter, fk_hook_id: param.hookId, - }) + }); - Tele.emit('evt', { evt_type: 'hookFilter:created' }) - return filter + T.emit('evt', { evt_type: 'hookFilter:created' }); + return filter; } - export async function hookFilterList(param: { hookId: any }) { - return Filter.rootFilterListByHook({ hookId: param.hookId }) + return Filter.rootFilterListByHook({ hookId: param.hookId }); } export async function filterDelete(param: { filterId: string }) { - await Filter.delete(param.filterId) - Tele.emit('evt', { evt_type: 'filter:deleted' }) + await Filter.delete(param.filterId); + T.emit('evt', { evt_type: 'filter:deleted' }); return true; } - export async function filterCreate(param: { - filter: FilterReqType - viewId: string + filter: FilterReqType; + viewId: string; }) { const filter = await Filter.insert({ ...param.filter, fk_view_id: param.viewId, }); - Tele.emit('evt', { evt_type: 'filter:created' }); + T.emit('evt', { evt_type: 'filter:created' }); - return filter + return filter; } export async function filterUpdate(param: { - filter: FilterReqType - filterId: string + filter: FilterReqType; + filterId: string; }) { - const filter = await Filter.update(param.filterId, param.filter) + // todo: type correction + const filter = await Filter.update(param.filterId, param.filter as Filter); + T.emit('evt', { evt_type: 'filter:updated' }); - Tele.emit('evt', { evt_type: 'filter:updated' }) - - return filter + return filter; } - export async function filterChildrenList(param: { filterId: any }) { return Filter.parentFilterList({ parentId: param.filterId, - }) + }); } - export async function filterGet(param: { filterId: string }) { - const filter = await Filter.get(param.filterId) - return filter + const filter = await Filter.get(param.filterId); + return filter; } export async function filterList(param: { viewId: string }) { - const filter = await Filter.rootFilterList({ viewId: param.viewId }) - return filter + const filter = await Filter.rootFilterList({ viewId: param.viewId }); + return filter; } diff --git a/packages/nocodb/src/lib/services/formViewColumnService.ts b/packages/nocodb/src/lib/services/formViewColumnService.ts index afef0e02f3..b91d171f3b 100644 --- a/packages/nocodb/src/lib/services/formViewColumnService.ts +++ b/packages/nocodb/src/lib/services/formViewColumnService.ts @@ -1,11 +1,11 @@ import { FormViewColumn } from '../models'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; export async function columnUpdate(param: { formViewColumnId: string; // todo: replace with FormColumnReq formViewColumn: FormViewColumn; }) { - Tele.emit('evt', { evt_type: 'formViewColumn:updated' }); + T.emit('evt', { evt_type: 'formViewColumn:updated' }); return await FormViewColumn.update( param.formViewColumnId, param.formViewColumn diff --git a/packages/nocodb/src/lib/services/formViewService.ts b/packages/nocodb/src/lib/services/formViewService.ts index d9a4667c72..eec4c66624 100644 --- a/packages/nocodb/src/lib/services/formViewService.ts +++ b/packages/nocodb/src/lib/services/formViewService.ts @@ -1,4 +1,4 @@ -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { FormReqType, ViewTypes } from 'nocodb-sdk'; import { FormView, View } from '../models'; @@ -11,7 +11,7 @@ export async function formViewCreate(param: { tableId: string; body: FormReqType; }) { - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'form' }); + T.emit('evt', { evt_type: 'vtable:created', show_as: 'form' }); const view = await View.insert({ ...param.body, // todo: sanitize @@ -26,6 +26,6 @@ export async function formViewUpdate(param: { formViewId: string; body: FormReqType; }) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'grid' }); + T.emit('evt', { evt_type: 'view:updated', type: 'grid' }); await FormView.update(param.formViewId, param.body); } diff --git a/packages/nocodb/src/lib/services/galleryViewService.ts b/packages/nocodb/src/lib/services/galleryViewService.ts index 9132202b74..9e8ab05d5d 100644 --- a/packages/nocodb/src/lib/services/galleryViewService.ts +++ b/packages/nocodb/src/lib/services/galleryViewService.ts @@ -1,6 +1,6 @@ import { GalleryReqType, ViewTypes } from 'nocodb-sdk'; -import { Tele } from 'nc-help'; -import { GalleryView, View } from '../models' +import { T } from 'nc-help'; +import { GalleryView, View } from '../models'; export async function galleryViewGet(param: { galleryViewId: string }) { return await GalleryView.get(param.galleryViewId); @@ -10,7 +10,7 @@ export async function galleryViewCreate(param: { tableId: string; gallery: GalleryReqType; }) { - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'gallery' }); + T.emit('evt', { evt_type: 'vtable:created', show_as: 'gallery' }); const view = await View.insert({ ...param.gallery, // todo: sanitize @@ -24,6 +24,6 @@ export async function galleryViewUpdate(param: { galleryViewId: string; gallery: GalleryReqType; }) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'gallery' }); + T.emit('evt', { evt_type: 'view:updated', type: 'gallery' }); await GalleryView.update(param.galleryViewId, param.gallery); } diff --git a/packages/nocodb/src/lib/services/gridViewColumnService.ts b/packages/nocodb/src/lib/services/gridViewColumnService.ts index 06731ec3b1..e24dd07177 100644 --- a/packages/nocodb/src/lib/services/gridViewColumnService.ts +++ b/packages/nocodb/src/lib/services/gridViewColumnService.ts @@ -1,6 +1,6 @@ import { GridColumnReqType } from 'nocodb-sdk'; import GridViewColumn from '../models/GridViewColumn'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; export async function columnList(param: { gridViewId: string }) { return await GridViewColumn.list(param.gridViewId); @@ -10,6 +10,6 @@ export async function gridColumnUpdate(param: { gridViewColumnId: string; grid: GridColumnReqType; }) { - Tele.emit('evt', { evt_type: 'gridViewColumn:updated' }); + T.emit('evt', { evt_type: 'gridViewColumn:updated' }); return await GridViewColumn.update(param.gridViewColumnId, param.grid); } diff --git a/packages/nocodb/src/lib/services/gridViewService.ts b/packages/nocodb/src/lib/services/gridViewService.ts index 55315d1fc8..c99f63552e 100644 --- a/packages/nocodb/src/lib/services/gridViewService.ts +++ b/packages/nocodb/src/lib/services/gridViewService.ts @@ -1,4 +1,4 @@ -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { GridReqType, ViewTypes } from 'nocodb-sdk'; import { View } from '../models'; import { GridView } from '../models'; @@ -13,7 +13,7 @@ export async function gridViewCreate(param: { fk_model_id: param.tableId, type: ViewTypes.GRID, }); - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'grid' }); + T.emit('evt', { evt_type: 'vtable:created', show_as: 'grid' }); return view; } @@ -21,6 +21,6 @@ export async function gridViewUpdate(param: { viewId: string; grid: GridReqType; }) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'grid' }); + T.emit('evt', { evt_type: 'view:updated', type: 'grid' }); return await GridView.update(param.viewId, param.grid); } diff --git a/packages/nocodb/src/lib/services/hookFilterService.ts b/packages/nocodb/src/lib/services/hookFilterService.ts index e8e94d7233..d4c0b2636d 100644 --- a/packages/nocodb/src/lib/services/hookFilterService.ts +++ b/packages/nocodb/src/lib/services/hookFilterService.ts @@ -1,4 +1,4 @@ -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { FilterReqType } from 'nocodb-sdk'; import Filter from '../models/Filter'; @@ -37,7 +37,7 @@ export async function filterCreate(param: { fk_hook_id: param.hookId, }); - Tele.emit('evt', { evt_type: 'hookFilter:created' }); + T.emit('evt', { evt_type: 'hookFilter:created' }); return filter; } @@ -50,12 +50,12 @@ export async function filterUpdate(param: { ...param.filter, fk_hook_id: param.hookId, } as Filter); - Tele.emit('evt', { evt_type: 'hookFilter:updated' }); + T.emit('evt', { evt_type: 'hookFilter:updated' }); return filter; } export async function filterDelete(param: { filterId: string }) { await Filter.delete(param.filterId); - Tele.emit('evt', { evt_type: 'hookFilter:deleted' }); + T.emit('evt', { evt_type: 'hookFilter:deleted' }); return true; } diff --git a/packages/nocodb/src/lib/services/hookService.ts b/packages/nocodb/src/lib/services/hookService.ts index 17cef05229..2e20732c17 100644 --- a/packages/nocodb/src/lib/services/hookService.ts +++ b/packages/nocodb/src/lib/services/hookService.ts @@ -1,4 +1,4 @@ -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { Hook, Model } from '../models'; import { HookReqType, HookTestReqType } from 'nocodb-sdk'; @@ -14,24 +14,26 @@ export async function hookCreate(param: { tableId: string; hook: HookReqType; }) { - Tele.emit('evt', { evt_type: 'webhooks:created' }); + T.emit('evt', { evt_type: 'webhooks:created' }); + // todo: type correction const hook = await Hook.insert({ ...param.hook, fk_model_id: param.tableId, - }); + } as any); return hook; } export async function hookDelete(param: { hookId: string }) { - Tele.emit('evt', { evt_type: 'webhooks:deleted' }); + T.emit('evt', { evt_type: 'webhooks:deleted' }); await Hook.delete(param.hookId); return true; } export async function hookUpdate(param: { hookId: string; hook: HookReqType }) { - Tele.emit('evt', { evt_type: 'webhooks:updated' }); + T.emit('evt', { evt_type: 'webhooks:updated' }); - return await Hook.update(param.hookId, param.hook); + // todo: correction in swagger + return await Hook.update(param.hookId, param.hook as any); } export async function hookTest(param: { @@ -53,7 +55,7 @@ export async function hookTest(param: { true ); - Tele.emit('evt', { evt_type: 'webhooks:tested' }); + T.emit('evt', { evt_type: 'webhooks:tested' }); return true; } diff --git a/packages/nocodb/src/lib/services/kanbanViewService.ts b/packages/nocodb/src/lib/services/kanbanViewService.ts index 6dc676e351..160044e26a 100644 --- a/packages/nocodb/src/lib/services/kanbanViewService.ts +++ b/packages/nocodb/src/lib/services/kanbanViewService.ts @@ -1,6 +1,6 @@ import { KanbanReqType, ViewTypes } from 'nocodb-sdk'; import { KanbanView, View } from '../models'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; export async function kanbanViewGet(param: { kanbanViewId: string }) { return await KanbanView.get(param.kanbanViewId); @@ -10,7 +10,7 @@ export async function kanbanViewCreate(param: { tableId: string; kanban: KanbanReqType; }) { - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'kanban' }); + T.emit('evt', { evt_type: 'vtable:created', show_as: 'kanban' }); const view = await View.insert({ ...param.kanban, // todo: sanitize @@ -24,6 +24,6 @@ export async function kanbanViewUpdate(param: { kanbanViewId: string; kanban: KanbanReqType; }) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'kanban' }); + T.emit('evt', { evt_type: 'view:updated', type: 'kanban' }); return await KanbanView.update(param.kanbanViewId, param.kanban); } diff --git a/packages/nocodb/src/lib/services/mapViewService.ts b/packages/nocodb/src/lib/services/mapViewService.ts index ba21af1b6c..199baa9d13 100644 --- a/packages/nocodb/src/lib/services/mapViewService.ts +++ b/packages/nocodb/src/lib/services/mapViewService.ts @@ -1,18 +1,18 @@ import { MapType, ViewTypes } from 'nocodb-sdk'; import View from '../models/View'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import MapView from '../models/MapView'; -export async function mapViewGet(param:{mapViewId: string}) { - return await MapView.get(param.mapViewId) +export async function mapViewGet(param: { mapViewId: string }) { + return await MapView.get(param.mapViewId); } -export async function mapViewCreate(param:{ - tableId: string, +export async function mapViewCreate(param: { + tableId: string; // todo: add MapReq in schema - map: MapType + map: MapType; }) { - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'map' }); + T.emit('evt', { evt_type: 'vtable:created', show_as: 'map' }); const view = await View.insert({ ...param.map, // todo: sanitize @@ -22,13 +22,12 @@ export async function mapViewCreate(param:{ return view; } -export async function mapViewUpdate(param:{ - mapViewId: string, +export async function mapViewUpdate(param: { + mapViewId: string; // todo: add MapReq in schema - map: MapType - + map: MapType; }) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'map' }); + T.emit('evt', { evt_type: 'view:updated', type: 'map' }); // todo: type correction - return await MapView.update(param.mapViewId, param.map as any) + return await MapView.update(param.mapViewId, param.map as any); } diff --git a/packages/nocodb/src/lib/services/metaDiffService.ts b/packages/nocodb/src/lib/services/metaDiffService.ts index 0786a0c19a..274ba8e705 100644 --- a/packages/nocodb/src/lib/services/metaDiffService.ts +++ b/packages/nocodb/src/lib/services/metaDiffService.ts @@ -1,6 +1,6 @@ // // Project CRUD -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import { isVirtualCol, ModelTypes, RelationTypes, UITypes } from 'nocodb-sdk'; import { @@ -576,9 +576,7 @@ export async function baseMetaDiff(param: { return changes; } -export async function metaDiffSync(param: { - projectId: string; -}) { +export async function metaDiffSync(param: { projectId: string }) { const project = await Project.getWithInfo(param.projectId); for (const base of project.bases) { const virtualColumnInsert: Array<() => Promise> = []; @@ -777,7 +775,7 @@ export async function metaDiffSync(param: { await extractAndGenerateManyToManyRelations(await base.getModels()); } - Tele.emit('evt', { evt_type: 'metaDiff:synced' }); + T.emit('evt', { evt_type: 'metaDiff:synced' }); return true; } @@ -969,7 +967,7 @@ export async function baseMetaDiffSync(param: { // populate m2m relations await extractAndGenerateManyToManyRelations(await base.getModels()); - Tele.emit('evt', { evt_type: 'baseMetaDiff:synced' }); + T.emit('evt', { evt_type: 'baseMetaDiff:synced' }); return true; } diff --git a/packages/nocodb/src/lib/services/modelVisibilityService.ts b/packages/nocodb/src/lib/services/modelVisibilityService.ts index ad71f02871..95471c0ead 100644 --- a/packages/nocodb/src/lib/services/modelVisibilityService.ts +++ b/packages/nocodb/src/lib/services/modelVisibilityService.ts @@ -2,13 +2,13 @@ import { VisibilityRuleReqType } from 'nocodb-sdk'; import { NcError } from '../meta/helpers/catchError'; import Model from '../models/Model'; import ModelRoleVisibility from '../models/ModelRoleVisibility'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; export async function xcVisibilityMetaSetAll(param: { visibilityRule: VisibilityRuleReqType; projectId: string; }) { - Tele.emit('evt', { evt_type: 'uiAcl:updated' }); + T.emit('evt', { evt_type: 'uiAcl:updated' }); for (const d of param.visibilityRule) { for (const role of Object.keys(d.disabled)) { const view = await Model.get(d.id); @@ -40,7 +40,7 @@ export async function xcVisibilityMetaSetAll(param: { } } } - Tele.emit('evt', { evt_type: 'uiAcl:updated' }); + T.emit('evt', { evt_type: 'uiAcl:updated' }); return true; } diff --git a/packages/nocodb/src/lib/services/orgLicenseService.ts b/packages/nocodb/src/lib/services/orgLicenseService.ts index 7de1f717bf..c39d7814a9 100644 --- a/packages/nocodb/src/lib/services/orgLicenseService.ts +++ b/packages/nocodb/src/lib/services/orgLicenseService.ts @@ -5,13 +5,11 @@ import Noco from '../Noco'; export async function licenseGet() { const license = await Store.get(NC_LICENSE_KEY); - return { key: license?.value } + return { key: license?.value }; } -export async function licenseSet(param:{ - key: string -}) { +export async function licenseSet(param: { key: string }) { await Store.saveOrUpdate({ value: param.key, key: NC_LICENSE_KEY }); await Noco.loadEEState(); - return true + return true; } diff --git a/packages/nocodb/src/lib/services/orgTokenService.ts b/packages/nocodb/src/lib/services/orgTokenService.ts index ede91f0bc6..30228c336a 100644 --- a/packages/nocodb/src/lib/services/orgTokenService.ts +++ b/packages/nocodb/src/lib/services/orgTokenService.ts @@ -1,7 +1,7 @@ import { ApiTokenReqType, OrgUserRoles } from 'nocodb-sdk'; import { User } from '../models'; import ApiToken from '../models/ApiToken'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { NcError } from '../meta/helpers/catchError'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; @@ -32,7 +32,7 @@ export async function apiTokenCreate(param: { user: User; apiToken: ApiTokenReqType; }) { - Tele.emit('evt', { evt_type: 'org:apiToken:created' }); + T.emit('evt', { evt_type: 'org:apiToken:created' }); return await ApiToken.insert({ ...param.apiToken, fk_user_id: param['user'].id, @@ -48,6 +48,6 @@ export async function apiTokenDelete(param: { user: User; token: string }) { ) { NcError.notFound('Token not found'); } - Tele.emit('evt', { evt_type: 'org:apiToken:deleted' }); + T.emit('evt', { evt_type: 'org:apiToken:deleted' }); return await ApiToken.delete(param.token); } diff --git a/packages/nocodb/src/lib/services/orgUserService.ts b/packages/nocodb/src/lib/services/orgUserService.ts index f58e85f890..318afea8c0 100644 --- a/packages/nocodb/src/lib/services/orgUserService.ts +++ b/packages/nocodb/src/lib/services/orgUserService.ts @@ -11,7 +11,7 @@ import { NC_APP_SETTINGS } from '../constants'; import { Audit, ProjectUser, Store, SyncSource, User } from '../models'; import Noco from '../Noco'; import { MetaTable } from '../utils/globals'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { NcError } from '../meta/helpers/catchError'; import { extractProps } from '../meta/helpers/extractProps'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; @@ -142,7 +142,7 @@ export async function userAdd(param: { }); const count = await User.count(); - Tele.emit('evt', { evt_type: 'org:user:invite', count }); + T.emit('evt', { evt_type: 'org:user:invite', count }); await Audit.insert({ op_type: AuditOperationTypes.ORG_USER, diff --git a/packages/nocodb/src/lib/services/pluginService.ts b/packages/nocodb/src/lib/services/pluginService.ts index b468e7240c..2cb885e001 100644 --- a/packages/nocodb/src/lib/services/pluginService.ts +++ b/packages/nocodb/src/lib/services/pluginService.ts @@ -1,30 +1,31 @@ -import { Tele } from 'nc-help'; -import { Plugin } from '../models' -import { PluginTestReqType, PluginType } from 'nocodb-sdk' +import { T } from 'nc-help'; +import { Plugin } from '../models'; +import { PluginTestReqType, PluginType } from 'nocodb-sdk'; import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; export async function pluginList() { - return await Plugin.list() + return await Plugin.list(); } -export async function pluginTest(param:{body: PluginTestReqType}) { - Tele.emit('evt', { evt_type: 'plugin:tested' }); - return await NcPluginMgrv2.test(param.body) +export async function pluginTest(param: { body: PluginTestReqType }) { + T.emit('evt', { evt_type: 'plugin:tested' }); + return await NcPluginMgrv2.test(param.body); } export async function pluginRead(param: { pluginId: string }) { - return await Plugin.get(param.pluginId) + return await Plugin.get(param.pluginId); } -export async function pluginUpdate( - param: { pluginId: string; plugin: PluginType } -) { +export async function pluginUpdate(param: { + pluginId: string; + plugin: PluginType; +}) { const plugin = await Plugin.update(param.pluginId, param.plugin); - Tele.emit('evt', { + T.emit('evt', { evt_type: plugin.active ? 'plugin:installed' : 'plugin:uninstalled', title: plugin.title, }); - return plugin + return plugin; } export async function isPluginActive(param: { pluginTitle: string }) { - return await Plugin.isPluginActive(param.pluginTitle) + return await Plugin.isPluginActive(param.pluginTitle); } diff --git a/packages/nocodb/src/lib/services/projectService.ts b/packages/nocodb/src/lib/services/projectService.ts index 131df14557..2218f21b34 100644 --- a/packages/nocodb/src/lib/services/projectService.ts +++ b/packages/nocodb/src/lib/services/projectService.ts @@ -1,5 +1,5 @@ import DOMPurify from 'isomorphic-dompurify'; -import { OrgUserRoles, ProjectReqType, } from 'nocodb-sdk'; +import { OrgUserRoles, ProjectReqType } from 'nocodb-sdk'; import { promisify } from 'util'; import { populateMeta } from '../meta/api/helpers'; import { extractPropsAndSanitize } from '../meta/helpers/extractProps'; @@ -95,23 +95,23 @@ export async function projectCreate(param: { for (const base of await project.getBases()) { const info = await populateMeta(base, project); - Tele.emit('evt_api_created', info); + T.emit('evt_api_created', info); delete base.config; } - Tele.emit('evt', { + T.emit('evt', { evt_type: 'project:created', xcdb: !projectBody.external, }); - Tele.emit('evt', { evt_type: 'project:rest' }); + T.emit('evt', { evt_type: 'project:rest' }); project; } const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 4); -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; export async function getProjectWithInfo(param: { projectId: string }) { const project = await Project.getWithInfo(param.projectId); @@ -120,7 +120,7 @@ export async function getProjectWithInfo(param: { projectId: string }) { export async function projectSoftDelete(param: { projectId: any }) { await Project.softDelete(param.projectId); - Tele.emit('evt', { evt_type: 'project:deleted' }); + T.emit('evt', { evt_type: 'project:deleted' }); return true; } @@ -152,7 +152,7 @@ export async function projectUpdate(param: { } const result = await Project.update(param.projectId, data); - Tele.emit('evt', { evt_type: 'project:update' }); + T.emit('evt', { evt_type: 'project:update' }); return result; } diff --git a/packages/nocodb/src/lib/services/projectUserService.ts b/packages/nocodb/src/lib/services/projectUserService.ts index 1ee50e9410..567568f488 100644 --- a/packages/nocodb/src/lib/services/projectUserService.ts +++ b/packages/nocodb/src/lib/services/projectUserService.ts @@ -1,5 +1,5 @@ import { OrgUserRoles, ProjectUserReqType } from 'nocodb-sdk'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import ProjectUser from '../models/ProjectUser'; import validator from 'validator'; @@ -109,7 +109,7 @@ export async function userInvite(param: { }); const count = await User.count(); - Tele.emit('evt', { evt_type: 'project:invite', count }); + T.emit('evt', { evt_type: 'project:invite', count }); await Audit.insert({ project_id: param.projectId, diff --git a/packages/nocodb/src/lib/services/sharedBaseService.ts b/packages/nocodb/src/lib/services/sharedBaseService.ts index 637a9e322c..53213d7627 100644 --- a/packages/nocodb/src/lib/services/sharedBaseService.ts +++ b/packages/nocodb/src/lib/services/sharedBaseService.ts @@ -1,4 +1,4 @@ -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { v4 as uuidv4 } from 'uuid'; import Project from '../models/Project'; import { NcError } from '../meta/helpers/catchError'; @@ -7,7 +7,7 @@ const config = { dashboardPath: '/nc', }; -export async function createSharedBaseLink(param:{ +export async function createSharedBaseLink(param: { projectId: string; roles: string; password: string; @@ -34,7 +34,7 @@ export async function createSharedBaseLink(param:{ data.url = `${param.siteUrl}${config.dashboardPath}#/nc/base/${data.uuid}`; delete data.password; - Tele.emit('evt', { evt_type: 'sharedBase:generated-link' }); + T.emit('evt', { evt_type: 'sharedBase:generated-link' }); return data; } @@ -64,11 +64,11 @@ export async function updateSharedBaseLink(param: { data.url = `${param.siteUrl}${config.dashboardPath}#/nc/base/${data.uuid}`; delete data.password; - Tele.emit('evt', { evt_type: 'sharedBase:generated-link' }); + T.emit('evt', { evt_type: 'sharedBase:generated-link' }); return data; } -export async function disableSharedBaseLink(param:{ +export async function disableSharedBaseLink(param: { projectId: string; }): Promise { const project = await Project.get(param.projectId); @@ -82,12 +82,12 @@ export async function disableSharedBaseLink(param:{ await Project.update(project.id, data); - Tele.emit('evt', { evt_type: 'sharedBase:disable-link' }); + T.emit('evt', { evt_type: 'sharedBase:disable-link' }); - return { uuid: null } + return { uuid: null }; } -export async function getSharedBaseLink(param:{ +export async function getSharedBaseLink(param: { projectId: string; siteUrl: string; }): Promise { @@ -105,4 +105,3 @@ export async function getSharedBaseLink(param:{ return data; } - diff --git a/packages/nocodb/src/lib/services/sortService.ts b/packages/nocodb/src/lib/services/sortService.ts index 4d69e278c8..e3931bad2c 100644 --- a/packages/nocodb/src/lib/services/sortService.ts +++ b/packages/nocodb/src/lib/services/sortService.ts @@ -1,21 +1,20 @@ import { SortReqType } from 'nocodb-sdk'; import Sort from '../models/Sort'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; export async function sortGet(param: { sortId: string }) { return Sort.get(param.sortId); } - export async function sortDelete(param: { sortId: string }) { await Sort.delete(param.sortId); - Tele.emit('evt', { evt_type: 'sort:deleted' }); + T.emit('evt', { evt_type: 'sort:deleted' }); return true; } export async function sortUpdate(param: { sortId: any; sort: SortReqType }) { const sort = await Sort.update(param.sortId, param.sort); - Tele.emit('evt', { evt_type: 'sort:updated' }); + T.emit('evt', { evt_type: 'sort:updated' }); return sort; } @@ -24,7 +23,7 @@ export async function sortCreate(param: { viewId: any; sort: SortReqType }) { ...param.sort, fk_view_id: param.viewId, } as Sort); - Tele.emit('evt', { evt_type: 'sort:created' }); + T.emit('evt', { evt_type: 'sort:created' }); return sort; } diff --git a/packages/nocodb/src/lib/services/swaggerService/getSwaggerJSON.ts b/packages/nocodb/src/lib/services/swaggerService/getSwaggerJSON.ts index 9ad9cdbcf6..8d4e81f478 100644 --- a/packages/nocodb/src/lib/services/swaggerService/getSwaggerJSON.ts +++ b/packages/nocodb/src/lib/services/swaggerService/getSwaggerJSON.ts @@ -1,4 +1,4 @@ -import { Model, Project, View } from '../../models' +import { Model, Project, View } from '../../models'; import FormViewColumn from '../../models/FormViewColumn'; import GalleryViewColumn from '../../models/GalleryViewColumn'; import Noco from '../../Noco'; diff --git a/packages/nocodb/src/lib/services/swaggerService/index.ts b/packages/nocodb/src/lib/services/swaggerService/index.ts index 7ee1c073d4..4b46830bf4 100644 --- a/packages/nocodb/src/lib/services/swaggerService/index.ts +++ b/packages/nocodb/src/lib/services/swaggerService/index.ts @@ -1,9 +1,12 @@ -import { NcError } from '../../meta/helpers/catchError' -import Model from '../../models/Model' -import Project from '../../models/Project' -import getSwaggerJSON from './getSwaggerJSON' +import { NcError } from '../../meta/helpers/catchError'; +import Model from '../../models/Model'; +import Project from '../../models/Project'; +import getSwaggerJSON from './getSwaggerJSON'; -export async function swaggerJson(param:{projectId:string; siteUrl:string}){ +export async function swaggerJson(param: { + projectId: string; + siteUrl: string; +}) { const project = await Project.get(param.projectId); if (!project) NcError.notFound(); diff --git a/packages/nocodb/src/lib/services/syncService/helpers/job.ts b/packages/nocodb/src/lib/services/syncService/helpers/job.ts index 80820cfeaf..b956a5353c 100644 --- a/packages/nocodb/src/lib/services/syncService/helpers/job.ts +++ b/packages/nocodb/src/lib/services/syncService/helpers/job.ts @@ -1,4 +1,4 @@ -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import FetchAT from './fetchAT'; import { UITypes } from 'nocodb-sdk'; // import * as sMap from './syncMap'; @@ -1933,7 +1933,7 @@ export default async ( }); } - Tele.event({ + T.event({ event: 'a:airtable-import:success', data: { stats: { @@ -2386,7 +2386,7 @@ export default async ( } } catch (e) { if (e.response?.data?.msg) { - Tele.event({ + T.event({ event: 'a:airtable-import:error', data: { error: e.response.data.msg }, }); diff --git a/packages/nocodb/src/lib/services/syncService/index.ts b/packages/nocodb/src/lib/services/syncService/index.ts index ddd27adcfd..1c1f17b177 100644 --- a/packages/nocodb/src/lib/services/syncService/index.ts +++ b/packages/nocodb/src/lib/services/syncService/index.ts @@ -1,6 +1,6 @@ -import { Tele } from 'nc-help'; -import { PagedResponseImpl } from '../../meta/helpers/PagedResponse' -import { Project, SyncSource } from '../../models' +import { T } from 'nc-help'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import { Project, SyncSource } from '../../models'; export async function syncSourceList(param: { projectId: string; @@ -18,7 +18,7 @@ export async function syncCreate(param: { // todo: define type syncPayload: Partial; }) { - Tele.emit('evt', { evt_type: 'webhooks:created' }); + T.emit('evt', { evt_type: 'webhooks:created' }); const project = await Project.getWithInfo(param.projectId); const sync = await SyncSource.insert({ @@ -31,17 +31,17 @@ export async function syncCreate(param: { } export async function syncDelete(param: { syncId: string }) { - Tele.emit('evt', { evt_type: 'webhooks:deleted' }); - return await SyncSource.delete(param.syncId) + T.emit('evt', { evt_type: 'webhooks:deleted' }); + return await SyncSource.delete(param.syncId); } -export async function syncUpdate(param:{ +export async function syncUpdate(param: { syncId: string; syncPayload: Partial; }) { - Tele.emit('evt', { evt_type: 'webhooks:updated' }); + T.emit('evt', { evt_type: 'webhooks:updated' }); - return await SyncSource.update(param.syncId, param.syncPayload) + return await SyncSource.update(param.syncId, param.syncPayload); } export { default as airtableImportJob } from './helpers/job'; diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts index a3459860fc..883927583f 100644 --- a/packages/nocodb/src/lib/services/tableService.ts +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -26,7 +26,7 @@ import Project from '../models/Project'; import User from '../models/User'; import View from '../models/View'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; export function reorderTable(param: { tableId: string; order: any }) { return Model.updateOrder(param.tableId, param.order); @@ -84,7 +84,7 @@ export async function tableDelete(param: { tableId: string; user: User }) { // ip: (req as any).clientIp, }).then(() => {}); - Tele.emit('evt', { evt_type: 'table:deleted' }); + T.emit('evt', { evt_type: 'table:deleted' }); return table.delete(); } @@ -319,8 +319,9 @@ export async function tableCreate(args: { mapDefaultDisplayValue(args.table.columns); - Tele.emit('evt', { evt_type: 'table:created' }); + T.emit('evt', { evt_type: 'table:created' }); + // todo: type correction const result = await Model.insert(project.id, base.id, { ...args.table, columns: columns.map((c, i) => { @@ -342,7 +343,7 @@ export async function tableCreate(args: { } as NormalColumnRequestType; }), order: +(tables?.pop()?.order ?? 0) + 1, - }); + } as any); return result; } diff --git a/packages/nocodb/src/lib/services/userService/helpers.ts b/packages/nocodb/src/lib/services/userService/helpers.ts index 4cfe0bd687..90e2d81dfc 100644 --- a/packages/nocodb/src/lib/services/userService/helpers.ts +++ b/packages/nocodb/src/lib/services/userService/helpers.ts @@ -1,7 +1,7 @@ import * as jwt from 'jsonwebtoken'; import crypto from 'crypto'; -import { NcConfig } from '../../../interface/config' -import { User } from '../../models' +import { NcConfig } from '../../../interface/config'; +import { User } from '../../models'; export function genJwt(user: User, config: NcConfig) { return jwt.sign( diff --git a/packages/nocodb/src/lib/services/userService/index.ts b/packages/nocodb/src/lib/services/userService/index.ts index 73b9a9948c..c18d35bd2e 100644 --- a/packages/nocodb/src/lib/services/userService/index.ts +++ b/packages/nocodb/src/lib/services/userService/index.ts @@ -1,46 +1,35 @@ -import { Request, Response } from 'express'; import { PasswordChangeReqType, - PasswordForgotReqType, PasswordResetReqType, - SignUpReqType, - TableType, + PasswordForgotReqType, + PasswordResetReqType, UserType, validatePassword, -} from 'nocodb-sdk' +} from 'nocodb-sdk'; import { OrgUserRoles } from 'nocodb-sdk'; -import { NC_APP_SETTINGS } from '../../../constants'; -import Store from '../../../models/Store'; -import { Tele } from 'nc-help'; -import catchError, { NcError } from '../../helpers/catchError'; +import { T } from 'nc-help'; -const { isEmail } = require('validator'); import * as ejs from 'ejs'; import bcrypt from 'bcryptjs'; import { promisify } from 'util'; -import User from '../../../models/User'; +import { NC_APP_SETTINGS } from '../../constants'; +import { NcError } from '../../meta/helpers/catchError'; +import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; +import { Audit, Store, User } from '../../models'; +import Noco from '../../Noco'; +import { MetaTable } from '../../utils/globals'; +import { randomTokenString } from './helpers'; const { v4: uuidv4 } = require('uuid'); -import Audit from '../../../models/Audit'; -import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2'; - -import passport from 'passport'; -import extractProjectIdAndAuthenticate from '../../helpers/extractProjectIdAndAuthenticate'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import { MetaTable } from '../../../utils/globals'; -import Noco from '../../../Noco'; -import { getAjvValidatorMw } from '../helpers'; -import { genJwt } from './helpers'; -import { randomTokenString } from '../../helpers/stringHelpers'; export async function registerNewUserIfAllowed({ - firstname, - lastname, - email, - salt, - password, - email_verification_token, - }: { + firstname, + lastname, + email, + salt, + password, + email_verification_token, +}: { firstname; lastname; email: string; @@ -54,7 +43,7 @@ export async function registerNewUserIfAllowed({ roles = `${OrgUserRoles.CREATOR},${OrgUserRoles.SUPER_ADMIN}`; // todo: update in nc_store // roles = 'owner,creator,editor' - Tele.emit('evt', { + T.emit('evt', { evt_type: 'project:invite', count: 1, }); @@ -85,11 +74,10 @@ export async function registerNewUserIfAllowed({ }); } - export async function passwordChange(param: { - body: PasswordChangeReqType - user: UserType - req:any + body: PasswordChangeReqType; + user: UserType; + req: any; }): Promise { const { currentPassword, newPassword } = param.body; @@ -133,13 +121,13 @@ export async function passwordChange(param: { ip: param.req?.clientIp, }); - return true + return true; } -export async function passwordForgot(param:{ +export async function passwordForgot(param: { body: PasswordForgotReqType; siteUrl: string; - req:any + req: any; }): Promise { const _email = param.body.email; @@ -165,9 +153,7 @@ export async function passwordForgot(param:{ adapter.mailSend({ to: user.email, subject: 'Password Reset Link', - text: `Visit following link to update your password : ${ - param.siteUrl - }/auth/password/reset/${token}.`, + text: `Visit following link to update your password : ${param.siteUrl}/auth/password/reset/${token}.`, html: ejs.render(template, { resetLink: param.siteUrl + `/auth/password/reset/${token}`, }), @@ -191,12 +177,10 @@ export async function passwordForgot(param:{ return NcError.badRequest('Your email has not been registered.'); } - return true + return true; } -export async function tokenValidate(param:{ - token: string; -}): Promise { +export async function tokenValidate(param: { token: string }): Promise { const token = param.token; const user = await Noco.ncMeta.metaGet(null, null, MetaTable.USERS, { @@ -210,14 +194,14 @@ export async function tokenValidate(param:{ NcError.badRequest('Password reset url expired'); } - return true + return true; } -export async function passwordReset(param:{ +export async function passwordReset(param: { body: PasswordResetReqType; token: string; // todo: exclude - req:any; + req: any; }): Promise { const { token, body, req } = param; @@ -261,7 +245,7 @@ export async function passwordReset(param:{ ip: req.clientIp, }); - return true + return true; } export async function emailVerification(param: { @@ -293,9 +277,8 @@ export async function emailVerification(param: { ip: req.clientIp, }); - return true + return true; } -export * from './helpers' -export * from './initAdminFromEnv' - +export * from './helpers'; +export { default as initAdminFromEnv } from './initAdminFromEnv'; diff --git a/packages/nocodb/src/lib/services/userService/initAdminFromEnv.ts b/packages/nocodb/src/lib/services/userService/initAdminFromEnv.ts index 8f9bfdce74..1a15b94649 100644 --- a/packages/nocodb/src/lib/services/userService/initAdminFromEnv.ts +++ b/packages/nocodb/src/lib/services/userService/initAdminFromEnv.ts @@ -1,15 +1,14 @@ -import User from '../../../models/User'; import { v4 as uuidv4 } from 'uuid'; import { promisify } from 'util'; import bcrypt from 'bcryptjs'; -import Noco from '../../../Noco'; -import { CacheScope, MetaTable } from '../../../utils/globals'; -import ProjectUser from '../../../models/ProjectUser'; import { validatePassword } from 'nocodb-sdk'; import boxen from 'boxen'; -import NocoCache from '../../../cache/NocoCache'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; +import NocoCache from '../../cache/NocoCache'; +import { ProjectUser, User } from '../../models'; +import Noco from '../../Noco'; +import { CacheScope, MetaTable } from '../../utils/globals'; const { isEmail } = require('validator'); const rolesLevel = { owner: 0, creator: 1, editor: 2, commenter: 3, viewer: 4 }; @@ -68,7 +67,7 @@ export default async function initAdminFromEnv(_ncMeta = Noco.ncMeta) { // if super admin not present if (await User.isFirst(ncMeta)) { // roles = 'owner,creator,editor' - Tele.emit('evt', { + T.emit('evt', { evt_type: 'project:invite', count: 1, }); @@ -126,7 +125,7 @@ export default async function initAdminFromEnv(_ncMeta = Noco.ncMeta) { ncMeta ); } else { - Tele.emit('evt', { + T.emit('evt', { evt_type: 'project:invite', count: 1, }); diff --git a/packages/nocodb/src/lib/services/userService/ui/auth/emailVerify.ts b/packages/nocodb/src/lib/services/userService/ui/auth/emailVerify.ts new file mode 100644 index 0000000000..f7412b12ba --- /dev/null +++ b/packages/nocodb/src/lib/services/userService/ui/auth/emailVerify.ts @@ -0,0 +1,70 @@ +export default ` + + + NocoDB - Verify Email + + + + + + + +
+ + + + + + Email verified successfully! + + + {{errMsg}} + + + + + + + +
+ + + + + +`; diff --git a/packages/nocodb/src/lib/services/userService/ui/auth/resetPassword.ts b/packages/nocodb/src/lib/services/userService/ui/auth/resetPassword.ts new file mode 100644 index 0000000000..514fc6d739 --- /dev/null +++ b/packages/nocodb/src/lib/services/userService/ui/auth/resetPassword.ts @@ -0,0 +1,108 @@ +export default ` + + + NocoDB - Reset Password + + + + + + + +
+ + + + + + Password reset successful! + + + + + + +
+ + + + + +`; diff --git a/packages/nocodb/src/lib/services/userService/ui/emailTemplates/forgotPassword.ts b/packages/nocodb/src/lib/services/userService/ui/emailTemplates/forgotPassword.ts new file mode 100644 index 0000000000..afb2f5849a --- /dev/null +++ b/packages/nocodb/src/lib/services/userService/ui/emailTemplates/forgotPassword.ts @@ -0,0 +1,171 @@ +export default ` + + + + + Simple Transactional Email + + + + + + + + + + + + + +`; diff --git a/packages/nocodb/src/lib/services/userService/ui/emailTemplates/invite.ts b/packages/nocodb/src/lib/services/userService/ui/emailTemplates/invite.ts new file mode 100644 index 0000000000..fc81f9409e --- /dev/null +++ b/packages/nocodb/src/lib/services/userService/ui/emailTemplates/invite.ts @@ -0,0 +1,208 @@ +export default ` + + + + + Simple Transactional Email + + + + + + + + + + + + + +`; diff --git a/packages/nocodb/src/lib/services/userService/ui/emailTemplates/verify.ts b/packages/nocodb/src/lib/services/userService/ui/emailTemplates/verify.ts new file mode 100644 index 0000000000..11702cc659 --- /dev/null +++ b/packages/nocodb/src/lib/services/userService/ui/emailTemplates/verify.ts @@ -0,0 +1,207 @@ +export default ` + + + + + Simple Transactional Email + + + + + + + + + + + + + +`; diff --git a/packages/nocodb/src/lib/services/utilService.ts b/packages/nocodb/src/lib/services/utilService.ts index 14c42094f1..b19eca2cf5 100644 --- a/packages/nocodb/src/lib/services/utilService.ts +++ b/packages/nocodb/src/lib/services/utilService.ts @@ -20,9 +20,7 @@ const versionCache = { lastFetched: null, }; - - -export async function testConnection(param:{body: any}) { +export async function testConnection(param: { body: any }) { return await SqlMgrv2.testConnection(param.body); } diff --git a/packages/nocodb/src/lib/services/viewColumnService.ts b/packages/nocodb/src/lib/services/viewColumnService.ts index f6825691ad..73d2041d46 100644 --- a/packages/nocodb/src/lib/services/viewColumnService.ts +++ b/packages/nocodb/src/lib/services/viewColumnService.ts @@ -1,5 +1,5 @@ -import { Tele } from 'nc-help'; -import { View } from '../models' +import { T } from 'nc-help'; +import { View } from '../models'; export async function columnList(param: { viewId: string }) { return await View.getColumns(param.viewId); @@ -18,7 +18,7 @@ export async function columnAdd(param: { view_id: param.viewId, } ); - Tele.emit('evt', { evt_type: 'viewColumn:inserted' }); + T.emit('evt', { evt_type: 'viewColumn:inserted' }); return viewColumn; } @@ -34,6 +34,6 @@ export async function columnUpdate(param: { param.columnId, param.column ); - Tele.emit('evt', { evt_type: 'viewColumn:updated' }); + T.emit('evt', { evt_type: 'viewColumn:updated' }); return result; } diff --git a/packages/nocodb/src/lib/services/viewService.ts b/packages/nocodb/src/lib/services/viewService.ts index 66b3aba3c4..f328d5bab8 100644 --- a/packages/nocodb/src/lib/services/viewService.ts +++ b/packages/nocodb/src/lib/services/viewService.ts @@ -1,6 +1,6 @@ +import { SharedViewType, ViewType } from 'nocodb-sdk'; import { Model, View } from '../models'; -import { Tele } from 'nc-help'; -import { SharedViewReqType, ViewReqType } from 'nocodb-sdk'; +import { T } from 'nc-help'; import { xcVisibilityMetaGet } from './modelVisibilityService'; export async function viewList(param: { @@ -11,12 +11,10 @@ export async function viewList(param: { }) { const model = await Model.get(param.tableId); - const viewList = await xcVisibilityMetaGet( - // param.projectId, - // param.baseId, - model.project_id, - [model] - ); + const viewList = await xcVisibilityMetaGet({ + projectId: model.project_id, + models: [model], + }); // todo: user roles //await View.list(param.tableId) @@ -31,34 +29,34 @@ export async function viewList(param: { // @ts-ignore export async function shareView(param: { viewId: string }) { - Tele.emit('evt', { evt_type: 'sharedView:generated-link' }); + T.emit('evt', { evt_type: 'sharedView:generated-link' }); return await View.share(param.viewId); } -// @ts-ignore -export async function viewUpdate(param: { viewId: string; view: ViewReqType }) { +// todo: type correctly +export async function viewUpdate(param: { viewId: string; view: ViewType }) { const result = await View.update(param.viewId, param.view); - Tele.emit('evt', { evt_type: 'vtable:updated', show_as: result.type }); + T.emit('evt', { evt_type: 'vtable:updated', show_as: result.type }); return result; } -// @ts-ignore export async function viewDelete(param: { viewId: string }) { await View.delete(param.viewId); - Tele.emit('evt', { evt_type: 'vtable:deleted' }); + T.emit('evt', { evt_type: 'vtable:deleted' }); return true; } export async function shareViewUpdate(param: { viewId: string; - sharedView: SharedViewReqType; + // todo: type correctly + sharedView: SharedViewType; }) { - Tele.emit('evt', { evt_type: 'sharedView:updated' }); + T.emit('evt', { evt_type: 'sharedView:updated' }); return await View.update(param.viewId, param.sharedView); } export async function shareViewDelete(param: { viewId: string }) { - Tele.emit('evt', { evt_type: 'sharedView:deleted' }); + T.emit('evt', { evt_type: 'sharedView:deleted' }); await View.sharedViewDelete(param.viewId); return true; } diff --git a/packages/nocodb/src/lib/utils/common/BaseApiBuilder.ts b/packages/nocodb/src/lib/utils/common/BaseApiBuilder.ts index 69294bdfc3..5b1f36e8a2 100644 --- a/packages/nocodb/src/lib/utils/common/BaseApiBuilder.ts +++ b/packages/nocodb/src/lib/utils/common/BaseApiBuilder.ts @@ -21,7 +21,7 @@ import NcProjectBuilder from '../../v1-legacy/NcProjectBuilder'; import Noco from '../../Noco'; import NcMetaIO from '../../meta/NcMetaIO'; import XcCache from '../../v1-legacy/plugins/adapters/cache/XcCache'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import BaseModel from './BaseModel'; import { XcCron } from './XcCron'; @@ -324,7 +324,7 @@ export default abstract class BaseApiBuilder } ); } - Tele.emit('evt', { evt_type: 'relation:created' }); + T.emit('evt', { evt_type: 'relation:created' }); } public async onRelationDelete( @@ -2979,7 +2979,7 @@ export default abstract class BaseApiBuilder } public async onTableCreate(_tn: string, _args?: any) { - Tele.emit('evt', { evt_type: 'table:created' }); + T.emit('evt', { evt_type: 'table:created' }); } public onVirtualTableUpdate(args: any) { diff --git a/packages/nocodb/src/lib/v1-legacy/NcProjectBuilder.ts b/packages/nocodb/src/lib/v1-legacy/NcProjectBuilder.ts index e155f40c15..15a63e4a6a 100644 --- a/packages/nocodb/src/lib/v1-legacy/NcProjectBuilder.ts +++ b/packages/nocodb/src/lib/v1-legacy/NcProjectBuilder.ts @@ -10,7 +10,7 @@ import SqlClientFactory from '../db/sql-client/lib/SqlClientFactory'; import Migrator from '../db/sql-migrator/lib/KnexMigrator'; import Noco from '../Noco'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import { GqlApiBuilder } from './gql/GqlApiBuilder'; import { XCEeError } from '../meta/NcMetaMgr'; import { RestApiBuilder } from './rest/RestApiBuilder'; @@ -840,7 +840,7 @@ export default class NcProjectBuilder { this.apiInfInfoList.push(info); this.aggregatedApiInfo = aggregatedInfo; if (isFirstTime) { - Tele.emit('evt_api_created', info); + T.emit('evt_api_created', info); } } diff --git a/packages/nocodb/src/lib/v1-legacy/rest/RestAuthCtrl.ts b/packages/nocodb/src/lib/v1-legacy/rest/RestAuthCtrl.ts index f78c17cd5c..1c8f61268c 100644 --- a/packages/nocodb/src/lib/v1-legacy/rest/RestAuthCtrl.ts +++ b/packages/nocodb/src/lib/v1-legacy/rest/RestAuthCtrl.ts @@ -29,7 +29,7 @@ const { isEmail } = require('validator'); import axios from 'axios'; import IEmailAdapter from '../../../interface/IEmailAdapter'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import XcCache from '../plugins/adapters/cache/XcCache'; passport.serializeUser(function ( @@ -130,7 +130,7 @@ export default class RestAuthCtrl { await this.createAuthTableIfNotExists(); await this.initStrategies(); - Tele.emit('evt_app_started', await this.users.count('id as count').first()); + T.emit('evt_app_started', await this.users.count('id as count').first()); this.app.router.use(passport.initialize()); const jwtMiddleware = passport.authenticate('jwt', { session: false }); @@ -406,7 +406,7 @@ export default class RestAuthCtrl { const token = req.query.state; if (token) { - Tele.emit('evt_subscribe', email); + T.emit('evt_subscribe', email); await this.users .update({ // firstname, lastname, @@ -431,7 +431,7 @@ export default class RestAuthCtrl { return cb({ msg: `Account not found!` }); } - Tele.emit('evt_subscribe', email); + T.emit('evt_subscribe', email); const salt = await promisify(bcrypt.genSalt)(10); user = await this.users.insert({ email: profile.emails[0].value, @@ -513,7 +513,7 @@ export default class RestAuthCtrl { const token = req.query?.state?.replace('github|', ''); if (token) { - Tele.emit('evt_subscribe', email); + T.emit('evt_subscribe', email); await this.users .update({ // firstname, lastname, @@ -538,7 +538,7 @@ export default class RestAuthCtrl { return cb({ msg: `Account not found!` }); } - Tele.emit('evt_subscribe', email); + T.emit('evt_subscribe', email); const salt = await promisify(bcrypt.genSalt)(10); user = await this.users.insert({ email: profile.emails[0].value, @@ -900,7 +900,7 @@ export default class RestAuthCtrl { const email_verification_token = uuidv4(); if (!ignore_subscribe) { - Tele.emit('evt_subscribe', email); + T.emit('evt_subscribe', email); } if (user) { @@ -928,7 +928,7 @@ export default class RestAuthCtrl { if (!(await this.users.first())) { // todo: update in nc_store // roles = 'owner,creator,editor' - Tele.emit('evt', { evt_type: 'project:invite', count: 1 }); + T.emit('evt', { evt_type: 'project:invite', count: 1 }); } else { if (process.env.NC_INVITE_ONLY_SIGNUP) { return next( @@ -1289,7 +1289,7 @@ export default class RestAuthCtrl { } } - Tele.emit('evt', { evt_type: 'project:invite', count: count?.count }); + T.emit('evt', { evt_type: 'project:invite', count: count?.count }); this.xcMeta.audit(req.body.project_id, null, 'nc_audit', { op_type: 'AUTHENTICATION', op_sub_type: 'INVITE', diff --git a/packages/nocodb/src/lib/v1-legacy/rest/RestAuthCtrlEE.ts b/packages/nocodb/src/lib/v1-legacy/rest/RestAuthCtrlEE.ts index a91eeb5558..efeae0b567 100644 --- a/packages/nocodb/src/lib/v1-legacy/rest/RestAuthCtrlEE.ts +++ b/packages/nocodb/src/lib/v1-legacy/rest/RestAuthCtrlEE.ts @@ -2,7 +2,7 @@ import passport from 'passport'; import { Strategy } from 'passport-jwt'; import { v4 as uuidv4 } from 'uuid'; import validator from 'validator'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import XcCache from '../plugins/adapters/cache/XcCache'; @@ -85,7 +85,7 @@ export default class RestAuthCtrlEE extends RestAuthCtrl { req.body.roles ); - Tele.emit('evt', { evt_type: 'project:invite', count: count?.count }); + T.emit('evt', { evt_type: 'project:invite', count: count?.count }); this.xcMeta.audit(req.body.project_id, null, 'nc_audit', { op_type: 'AUTHENTICATION', diff --git a/packages/nocodb/src/lib/version-upgrader/NcUpgrader.ts b/packages/nocodb/src/lib/version-upgrader/NcUpgrader.ts index e4354301eb..7a1c945f22 100644 --- a/packages/nocodb/src/lib/version-upgrader/NcUpgrader.ts +++ b/packages/nocodb/src/lib/version-upgrader/NcUpgrader.ts @@ -2,7 +2,7 @@ import { NcConfig } from '../../interface/config'; import debug from 'debug'; import NcMetaIO from '../meta/NcMetaIO'; -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import ncProjectEnvUpgrader from './ncProjectEnvUpgrader'; import ncProjectEnvUpgrader0011045 from './ncProjectEnvUpgrader0011045'; import ncProjectUpgraderV2_0090000 from './ncProjectUpgraderV2_0090000'; @@ -107,14 +107,14 @@ export default class NcUpgrader { } } await ctx.ncMeta.commit(); - Tele.emit('evt', { + T.emit('evt', { evt_type: 'appMigration:upgraded', from: oldVersion, to: process.env.NC_VERSION, }); } catch (e) { await ctx.ncMeta.rollback(e); - Tele.emit('evt', { + T.emit('evt', { evt_type: 'appMigration:failed', from: oldVersion, to: process.env.NC_VERSION, From db1a9984b263c66462e2f651ba77f67c76737ae1 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 14:00:11 +0530 Subject: [PATCH 24/64] chore: build sdk Signed-off-by: Pranav C --- packages/nocodb-sdk/src/lib/Api.ts | 52 +++++++++++++++--------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/Api.ts b/packages/nocodb-sdk/src/lib/Api.ts index a1017e4b07..c2197a21be 100644 --- a/packages/nocodb-sdk/src/lib/Api.ts +++ b/packages/nocodb-sdk/src/lib/Api.ts @@ -651,7 +651,7 @@ export interface HookReqType { env?: string; event: 'after' | 'before'; operation: 'insert' | 'delete' | 'update'; - async?: BoolType; + async?: string | number | null; notification: object; retries?: number; retry_interval?: number; @@ -1095,8 +1095,8 @@ export class Api< > extends HttpClient { auth = { /** - * @description Create a new user with provided email and password and first user is marked as super admin. - * + * @description Create a new user with provided email and password and first user is marked as super admin. + * * @tags Auth * @name Signup * @summary Signup @@ -1130,7 +1130,7 @@ export class Api< /** * @description Clear refresh token from the database and cookie. - * + * * @tags Auth * @name Signout * @summary Signout @@ -1154,8 +1154,8 @@ export class Api< }), /** - * @description Authenticate existing user with their email and password. Successful login will return a JWT access-token. - * + * @description Authenticate existing user with their email and password. Successful login will return a JWT access-token. + * * @tags Auth * @name Signin * @summary Signin @@ -1231,7 +1231,7 @@ export class Api< /** * @description Change password of authenticated user with a new one. - * + * * @tags Auth * @name PasswordChange * @summary Password change @@ -1334,7 +1334,7 @@ export class Api< /** * No description - * + * * @tags Auth * @name ProjectUserList * @summary Project users @@ -1455,7 +1455,7 @@ export class Api< orgTokens = { /** * No description - * + * * @tags Org tokens * @name List * @summary Organisation API Tokens List @@ -1525,7 +1525,7 @@ export class Api< orgLicense = { /** * No description - * + * * @tags Org license * @name Get * @summary App license get @@ -1569,7 +1569,7 @@ export class Api< orgAppSettings = { /** * No description - * + * * @tags Org app settings * @name Get * @summary App settings get @@ -1618,7 +1618,7 @@ export class Api< orgUsers = { /** * No description - * + * * @tags Org users * @name List * @summary Organisation Users @@ -1723,7 +1723,7 @@ export class Api< /** * No description - * + * * @tags Org users * @name GeneratePasswordResetToken * @summary Organisation User Generate Password Reset Token @@ -1751,7 +1751,7 @@ export class Api< project = { /** * No description - * + * * @tags Project * @name MetaGet * @summary Project info @@ -1936,7 +1936,7 @@ export class Api< /** * @description Read project details - * + * * @tags Project * @name SharedBaseGet * @request GET:/api/v1/db/meta/projects/{projectId}/shared @@ -2004,7 +2004,7 @@ export class Api< /** * No description - * + * * @tags Project * @name SharedBaseUpdate * @request PATCH:/api/v1/db/meta/projects/{projectId}/shared @@ -2106,7 +2106,7 @@ export class Api< /** * No description - * + * * @tags Project * @name AuditList * @request GET:/api/v1/db/meta/projects/{projectId}/audits @@ -2998,7 +2998,7 @@ export class Api< /** * No description - * + * * @tags DB view share * @name Create * @request POST:/api/v1/db/meta/views/{viewId}/share @@ -3117,7 +3117,7 @@ export class Api< dbTableSort = { /** * No description - * + * * @tags DB table sort * @name List * @request GET:/api/v1/db/meta/views/{viewId}/sorts @@ -4318,7 +4318,7 @@ export class Api< /** * @description Read project details - * + * * @tags Public * @name SharedBaseGet * @request GET:/api/v1/db/public/shared-base/{sharedBaseUuid}/meta @@ -4342,7 +4342,7 @@ export class Api< /** * No description - * + * * @tags Public * @name SharedViewMetaGet * @request GET:/api/v1/db/public/shared-view/{sharedViewUuid}/meta @@ -4475,7 +4475,7 @@ export class Api< /** * No description - * + * * @tags Utils * @name TestConnection * @request POST:/api/v1/db/meta/connection/test @@ -4585,7 +4585,7 @@ export class Api< /** * No description - * + * * @tags Utils * @name AggregatedMetaInfo * @request GET:/api/v1/aggregated-meta-info @@ -4702,7 +4702,7 @@ export class Api< dbTableWebhook = { /** * No description - * + * * @tags DB table webhook * @name List * @request GET:/api/v1/db/meta/tables/{tableId}/hooks @@ -4768,7 +4768,7 @@ export class Api< /** * No description - * + * * @tags DB table webhook * @name SamplePayloadGet * @request GET:/api/v1/db/meta/tables/{tableId}/hooks/samplePayload/{operation} @@ -4837,7 +4837,7 @@ export class Api< plugin = { /** * No description - * + * * @tags Plugin * @name List * @request GET:/api/v1/db/meta/plugins From 0cb2d13701135b549197ebd93434a40899d5d408 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 16:10:35 +0530 Subject: [PATCH 25/64] fix: add missing return statement Signed-off-by: Pranav C --- packages/nocodb/src/lib/controllers/tableController.ts | 2 +- packages/nocodb/src/lib/services/dataService/index.ts | 2 +- packages/nocodb/src/lib/services/projectService.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index df80818bff..cc8ace772b 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -22,7 +22,7 @@ export async function tableList(req: Request, res: Response) { } export async function tableCreate(req: Request, res) { - const result = tableService.tableCreate({ + const result = await tableService.tableCreate({ projectId: req.params.projectId, baseId: req.params.baseId, table: req.body, diff --git a/packages/nocodb/src/lib/services/dataService/index.ts b/packages/nocodb/src/lib/services/dataService/index.ts index 44e4b7bedb..40cbddab0c 100644 --- a/packages/nocodb/src/lib/services/dataService/index.ts +++ b/packages/nocodb/src/lib/services/dataService/index.ts @@ -56,7 +56,7 @@ export async function dataInsert( dbDriver: NcConnectionMgrv2.get(base), }); - await baseModel.insert(param.body, null, param.cookie); + return await baseModel.insert(param.body, null, param.cookie); } export async function dataUpdate( diff --git a/packages/nocodb/src/lib/services/projectService.ts b/packages/nocodb/src/lib/services/projectService.ts index 2218f21b34..eaf4b98a64 100644 --- a/packages/nocodb/src/lib/services/projectService.ts +++ b/packages/nocodb/src/lib/services/projectService.ts @@ -106,7 +106,7 @@ export async function projectCreate(param: { T.emit('evt', { evt_type: 'project:rest' }); - project; + return project; } const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz_', 4); From 5a3298403f9e1a5dacf85781801d7c60a1fc7055 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 16:43:05 +0530 Subject: [PATCH 26/64] fix: typo correction Signed-off-by: Pranav C --- packages/nocodb/src/lib/controllers/dataController/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/controllers/dataController/index.ts b/packages/nocodb/src/lib/controllers/dataController/index.ts index d2c11c8a92..b9e7207d9a 100644 --- a/packages/nocodb/src/lib/controllers/dataController/index.ts +++ b/packages/nocodb/src/lib/controllers/dataController/index.ts @@ -32,7 +32,7 @@ async function dataFindOne(req: Request, res: Response) { async function dataGroupBy(req: Request, res: Response) { res.json( - await dataService.dataFindOne({ + await dataService.dataGroupBy({ query: req.query, projectName: req.params.projectName, tableName: req.params.tableName, From f4c0ce50abbd398c4e3f9661d6549983716ba02f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 17:20:39 +0530 Subject: [PATCH 27/64] fix: api handler order correction Signed-off-by: Pranav C --- .../src/lib/controllers/dataApis/index.ts | 2 +- .../dataController/dataAliasNestedApis.ts | 291 ++++++++++++++++++ .../src/lib/services/dataService/index.ts | 6 +- 3 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/index.ts b/packages/nocodb/src/lib/controllers/dataApis/index.ts index 34bb194ee0..3a4528a5e1 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/index.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/index.ts @@ -2,7 +2,7 @@ import dataApis from './dataApis'; import oldDataApis from './oldDataApis'; import dataAliasApis from './dataAliasApis'; import bulkDataAliasApis from './bulkDataAliasApis'; -import dataAliasNestedApis from './dataAliasNestedApis'; +import dataAliasNestedApis from '../../../controllers/dataController/dataAliasNestedApis'; import dataAliasExportApis from './dataAliasExportApis'; export { diff --git a/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts b/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts new file mode 100644 index 0000000000..eb11f56d45 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts @@ -0,0 +1,291 @@ +import { Request, Response, Router } from 'express'; +import Model from '../../models/Model'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import { + getColumnByIdOrName, + getViewAndModelFromRequestByAliasOrId, +} from '../../meta/api/dataApis/helpers'; +import { NcError } from '../../meta/helpers/catchError'; +import apiMetrics from '../../meta/helpers/apiMetrics'; + +// todo: handle case where the given column is not ltar +export async function mmList(req: Request, res: Response, next) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + + if (!model) return next(new Error('Table not found')); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(req.params.columnName, model); + + const data = await baseModel.mmList( + { + colId: column.id, + parentId: req.params.rowId, + }, + req.query as any + ); + const count: any = await baseModel.mmListCount({ + colId: column.id, + parentId: req.params.rowId, + }); + + res.json( + new PagedResponseImpl(data, { + count, + ...req.query, + }) + ); +} + +export async function mmExcludedList(req: Request, res: Response, next) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + if (!model) return next(new Error('Table not found')); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + const column = await getColumnByIdOrName(req.params.columnName, model); + + const data = await baseModel.getMmChildrenExcludedList( + { + colId: column.id, + pid: req.params.rowId, + }, + req.query + ); + + const count = await baseModel.getMmChildrenExcludedListCount( + { + colId: column.id, + pid: req.params.rowId, + }, + req.query + ); + + res.json( + new PagedResponseImpl(data, { + count, + ...req.query, + }) + ); +} + +export async function hmExcludedList(req: Request, res: Response, next) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + + if (!model) return next(new Error('Table not found')); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(req.params.columnName, model); + + const data = await baseModel.getHmChildrenExcludedList( + { + colId: column.id, + pid: req.params.rowId, + }, + req.query + ); + + const count = await baseModel.getHmChildrenExcludedListCount( + { + colId: column.id, + pid: req.params.rowId, + }, + req.query + ); + + res.json( + new PagedResponseImpl(data, { + count, + ...req.query, + }) + ); +} + +export async function btExcludedList(req: Request, res: Response, next) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + if (!model) return next(new Error('Table not found')); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(req.params.columnName, model); + + const data = await baseModel.getBtChildrenExcludedList( + { + colId: column.id, + cid: req.params.rowId, + }, + req.query + ); + + const count = await baseModel.getBtChildrenExcludedListCount( + { + colId: column.id, + cid: req.params.rowId, + }, + req.query + ); + + res.json( + new PagedResponseImpl(data, { + count, + ...req.query, + }) + ); +} + +// todo: handle case where the given column is not ltar +export async function hmList(req: Request, res: Response, next) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + if (!model) return next(new Error('Table not found')); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(req.params.columnName, model); + + const data = await baseModel.hmList( + { + colId: column.id, + id: req.params.rowId, + }, + req.query + ); + + const count = await baseModel.hmListCount({ + colId: column.id, + id: req.params.rowId, + }); + + res.json( + new PagedResponseImpl(data, { + count, + ...req.query, + } as any) + ); +} + +//@ts-ignore +async function relationDataRemove(req, res) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(req.params.columnName, model); + + await baseModel.removeChild({ + colId: column.id, + childId: req.params.refRowId, + rowId: req.params.rowId, + cookie: req, + }); + + res.json({ msg: 'success' }); +} + +//@ts-ignore +// todo: Give proper error message when reference row is already related and handle duplicate ref row id in hm +async function relationDataAdd(req, res) { + const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(req.params.columnName, model); + await baseModel.addChild({ + colId: column.id, + childId: req.params.refRowId, + rowId: req.params.rowId, + cookie: req, + }); + + res.json({ msg: 'success' }); +} + +const router = Router({ mergeParams: true }); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/mm/:columnName/exclude', + apiMetrics, + ncMetaAclMw(mmExcludedList, 'mmExcludedList') +); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/hm/:columnName/exclude', + apiMetrics, + ncMetaAclMw(hmExcludedList, 'hmExcludedList') +); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/bt/:columnName/exclude', + apiMetrics, + ncMetaAclMw(btExcludedList, 'btExcludedList') +); + +router.post( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/:relationType/:columnName/:refRowId', + apiMetrics, + ncMetaAclMw(relationDataAdd, 'relationDataAdd') +); +router.delete( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/:relationType/:columnName/:refRowId', + apiMetrics, + ncMetaAclMw(relationDataRemove, 'relationDataRemove') +); + +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/mm/:columnName', + apiMetrics, + ncMetaAclMw(mmList, 'mmList') +); +router.get( + '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/hm/:columnName', + apiMetrics, + ncMetaAclMw(hmList, 'hmList') +); + +export default router; diff --git a/packages/nocodb/src/lib/services/dataService/index.ts b/packages/nocodb/src/lib/services/dataService/index.ts index 40cbddab0c..1f1d09b20a 100644 --- a/packages/nocodb/src/lib/services/dataService/index.ts +++ b/packages/nocodb/src/lib/services/dataService/index.ts @@ -19,7 +19,7 @@ export async function dataFindOne(param: PathParams & { query: any }) { export async function dataGroupBy(param: PathParams & { query: any }) { const { model, view } = await getViewAndModelByAliasOrId(param); - await getDataGroupBy({ model, view, query: param.query }); + return await getDataGroupBy({ model, view, query: param.query }); } export async function dataCount(param: PathParams & { query: any }) { @@ -245,7 +245,7 @@ export async function dataExist( dbDriver: NcConnectionMgrv2.get(base), }); - await baseModel.exist(param.rowId); + return await baseModel.exist(param.rowId); } // todo: Handle the error case where view doesnt belong to model @@ -508,7 +508,7 @@ export async function hmExcludedList(param: { param.query ); - new PagedResponseImpl(data, { + return new PagedResponseImpl(data, { count, ...param.query, }); From 1f9d36a9f39ec2729606cac60773c1c1fdcf0124 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 17:24:30 +0530 Subject: [PATCH 28/64] fix: remove extra object wrapping Signed-off-by: Pranav C --- packages/nocodb/src/lib/controllers/dataController/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/dataController/index.ts b/packages/nocodb/src/lib/controllers/dataController/index.ts index b9e7207d9a..522ed1be5e 100644 --- a/packages/nocodb/src/lib/controllers/dataController/index.ts +++ b/packages/nocodb/src/lib/controllers/dataController/index.ts @@ -42,14 +42,14 @@ async function dataGroupBy(req: Request, res: Response) { } async function dataCount(req: Request, res: Response) { - const count = await dataService.dataCount({ + const countResult = await dataService.dataCount({ query: req.query, projectName: req.params.projectName, tableName: req.params.tableName, viewName: req.params.viewName, }); - res.json({ count }); + res.json(countResult); } async function dataInsert(req: Request, res: Response) { From 9a1970a32235c016d69fb355f0b0c6e183a92d9f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 17:43:18 +0530 Subject: [PATCH 29/64] fix: handle not null error Signed-off-by: Pranav C --- .../nocodb/src/lib/meta/helpers/catchError.ts | 15 +++++++++++++++ .../nocodb/tests/unit/rest/tests/tableRow.test.ts | 8 ++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/meta/helpers/catchError.ts b/packages/nocodb/src/lib/meta/helpers/catchError.ts index 1c3393ed0d..fdf1344638 100644 --- a/packages/nocodb/src/lib/meta/helpers/catchError.ts +++ b/packages/nocodb/src/lib/meta/helpers/catchError.ts @@ -5,6 +5,7 @@ enum DBError { COLUMN_NOT_EXIST = 'COLUMN_NOT_EXIST', CONSTRAINT_EXIST = 'CONSTRAINT_EXIST', CONSTRAINT_NOT_EXIST = 'CONSTRAINT_NOT_EXIST', + COLUMN_NOT_NULL = 'COLUMN_NOT_NULL', } // extract db errors using database error code @@ -20,6 +21,7 @@ function extractDBError(error): { let extra: Record; let type: DBError; + // todo: handle not null constraint error for all databases switch (error.code) { // sqlite errors case 'SQLITE_BUSY': @@ -172,6 +174,19 @@ function extractDBError(error): { break; case 'ER_BAD_NULL_ERROR': message = 'A null value is not allowed for this field.'; + { + const extractColNameMatch = error.message.match( + /Column '(\w+)' cannot be null/i + ); + if (extractColNameMatch && extractColNameMatch[1]) { + message = `The column '${extractColNameMatch[1]}' cannot be null.`; + type = DBError.COLUMN_NOT_NULL; + extra = { + column: extractColNameMatch[1], + }; + } + } + break; case 'ER_DATA_TOO_LONG': message = 'The data entered is too long for this field.'; diff --git a/packages/nocodb/tests/unit/rest/tests/tableRow.test.ts b/packages/nocodb/tests/unit/rest/tests/tableRow.test.ts index 3f2b75e7e0..9fdc944bd2 100644 --- a/packages/nocodb/tests/unit/rest/tests/tableRow.test.ts +++ b/packages/nocodb/tests/unit/rest/tests/tableRow.test.ts @@ -2125,9 +2125,13 @@ function tableTest() { .set('xc-auth', context.token) .expect(400); + // todo: only keep generic error message once updated in noco catchError middleware if ( - !response.body.msg.includes("Column 'customer_id' cannot be null") && - !response.body.msg.includes('Cannot add or update a child row') + !response.body.message?.includes("The column 'customer_id' cannot be null") && + !response.body.message?.includes("Column 'customer_id' cannot be null") && + !response.body.message?.includes('Cannot add or update a child row') && + !response.body.msg?.includes("Column 'customer_id' cannot be null") && + !response.body.msg?.includes('Cannot add or update a child row') ) { console.log( 'Delete list hm with existing ref row id with non nullable clause', From 4dd5ed20b6fc2aedd8acc10c5531f27ea669f45c Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 18:49:41 +0530 Subject: [PATCH 30/64] fix: path param typo correction Signed-off-by: Pranav C --- packages/nocodb/src/lib/controllers/public/publicMetaApis.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/controllers/public/publicMetaApis.ts b/packages/nocodb/src/lib/controllers/public/publicMetaApis.ts index a6b261625e..238e3223ae 100644 --- a/packages/nocodb/src/lib/controllers/public/publicMetaApis.ts +++ b/packages/nocodb/src/lib/controllers/public/publicMetaApis.ts @@ -13,7 +13,7 @@ export async function viewMetaGet(req: Request, res: Response) { async function publicSharedBaseGet(req, res): Promise { res.json( await publicMetaService.publicSharedBaseGet({ - sharedBaseUuid: req.params.sharedViewUuid, + sharedBaseUuid: req.params.sharedBaseUuid, }) ); } From 610ba3c4d4b0843a9588fcda235e529cc0a10192 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 18:53:09 +0530 Subject: [PATCH 31/64] fix: add missing await Signed-off-by: Pranav C --- packages/nocodb/src/lib/controllers/public/publicDataApis.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/public/publicDataApis.ts b/packages/nocodb/src/lib/controllers/public/publicDataApis.ts index 77ba40c157..6a77be7c01 100644 --- a/packages/nocodb/src/lib/controllers/public/publicDataApis.ts +++ b/packages/nocodb/src/lib/controllers/public/publicDataApis.ts @@ -5,7 +5,7 @@ import catchError from '../../meta/helpers/catchError'; import { publicDataService } from '../../services'; export async function dataList(req: Request, res: Response) { - const pagedResponse = publicDataService.dataList({ + const pagedResponse = await publicDataService.dataList({ query: req.query, password: req.headers?.['xc-password'] as string, sharedViewUuid: req.params.sharedViewUuid, @@ -36,7 +36,7 @@ async function dataInsert(req: Request & { files: any[] }, res: Response) { } async function relDataList(req, res) { - const pagedResponse = publicDataService.relDataList({ + const pagedResponse = await publicDataService.relDataList({ query: req.query, password: req.headers?.['xc-password'] as string, sharedViewUuid: req.params.sharedViewUuid, From e6116cf7ef47b35ff49594da8cc3e9b2c5a4e3db Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 19:02:18 +0530 Subject: [PATCH 32/64] fix: replace model with view Signed-off-by: Pranav C --- packages/nocodb/src/lib/services/modelVisibilityService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/services/modelVisibilityService.ts b/packages/nocodb/src/lib/services/modelVisibilityService.ts index 95471c0ead..eb2f21ec7e 100644 --- a/packages/nocodb/src/lib/services/modelVisibilityService.ts +++ b/packages/nocodb/src/lib/services/modelVisibilityService.ts @@ -1,8 +1,8 @@ import { VisibilityRuleReqType } from 'nocodb-sdk'; import { NcError } from '../meta/helpers/catchError'; -import Model from '../models/Model'; import ModelRoleVisibility from '../models/ModelRoleVisibility'; import { T } from 'nc-help'; +import { Model, View } from '../models'; export async function xcVisibilityMetaSetAll(param: { visibilityRule: VisibilityRuleReqType; @@ -11,7 +11,7 @@ export async function xcVisibilityMetaSetAll(param: { T.emit('evt', { evt_type: 'uiAcl:updated' }); for (const d of param.visibilityRule) { for (const role of Object.keys(d.disabled)) { - const view = await Model.get(d.id); + const view = await View.get(d.id); if (view.project_id !== param.projectId) { NcError.badRequest('View does not belong to the project'); From 2f80ac4530ecf307f6abe4f5425a614f7e9055bc Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 19:35:14 +0530 Subject: [PATCH 33/64] fix: public data api response correction Signed-off-by: Pranav C --- packages/nocodb/src/lib/controllers/public/publicDataApis.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/controllers/public/publicDataApis.ts b/packages/nocodb/src/lib/controllers/public/publicDataApis.ts index 6a77be7c01..46a73a0f2a 100644 --- a/packages/nocodb/src/lib/controllers/public/publicDataApis.ts +++ b/packages/nocodb/src/lib/controllers/public/publicDataApis.ts @@ -10,7 +10,7 @@ export async function dataList(req: Request, res: Response) { password: req.headers?.['xc-password'] as string, sharedViewUuid: req.params.sharedViewUuid, }); - res.json(pagedResponse); + res.json({ data: pagedResponse }); } // todo: Handle the error case where view doesnt belong to model @@ -23,6 +23,7 @@ async function groupedDataList(req: Request, res: Response) { }); res.json(groupedData); } + async function dataInsert(req: Request & { files: any[] }, res: Response) { const insertResult = await publicDataService.dataInsert({ sharedViewUuid: req.params.sharedViewUuid, From 8f455b9b8ca3d5381dc00542ce79496981cd9652 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 2 Mar 2023 19:36:21 +0530 Subject: [PATCH 34/64] fix: ajv validator function and ajv exception handler Signed-off-by: Pranav C --- .../src/lib/meta/api/helpers/apiHelpers.ts | 22 ++++++++++++++++++- .../nocodb/src/lib/meta/helpers/catchError.ts | 17 ++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/meta/api/helpers/apiHelpers.ts b/packages/nocodb/src/lib/meta/api/helpers/apiHelpers.ts index 49d1fbc462..25a077a52d 100644 --- a/packages/nocodb/src/lib/meta/api/helpers/apiHelpers.ts +++ b/packages/nocodb/src/lib/meta/api/helpers/apiHelpers.ts @@ -2,6 +2,7 @@ import { NextFunction, Request, Response } from 'express'; import Ajv, { ErrorObject } from 'ajv'; // @ts-ignore import swagger from '../../../../schema/swagger.json'; +import { NcError } from '../../helpers/catchError' export function parseHrtimeToSeconds(hrtime) { const seconds = (hrtime[0] + hrtime[1] / 1e6).toFixed(3); @@ -29,10 +30,29 @@ export const getAjvValidatorMw = (schema) => { // If the request body is invalid, send a response with an error message res.status(400).json({ - status: 'error', message: 'Invalid request body', errors, }); } }; }; + +// a function to validate the payload against the schema +export const ajvValidator = (schema, payload) => { + // Validate the request body against the schema + const valid = ajv.validate( + typeof schema === 'string' ? { $ref: schema } : schema, + payload + ); + + // If the request body is not valid, throw error + if (!valid) { + const errors: ErrorObject[] | null | undefined = ajv.errors; + + // If the request body is invalid, throw error with error message and errors + NcError.ajvValidationError({ + message: 'Invalid request body', + errors, + }); + } +} diff --git a/packages/nocodb/src/lib/meta/helpers/catchError.ts b/packages/nocodb/src/lib/meta/helpers/catchError.ts index fdf1344638..f7d6f9dd4b 100644 --- a/packages/nocodb/src/lib/meta/helpers/catchError.ts +++ b/packages/nocodb/src/lib/meta/helpers/catchError.ts @@ -1,3 +1,5 @@ +import { ErrorObject } from 'ajv'; + enum DBError { TABLE_EXIST = 'TABLE_EXIST', TABLE_NOT_EXIST = 'TABLE_NOT_EXIST', @@ -409,6 +411,8 @@ export default function ( return res.status(500).json({ msg: e.message }); } else if (e instanceof NotImplemented) { return res.status(501).json({ msg: e.message }); + } else if (e instanceof AjvError) { + return res.status(501).json({ msg: e.message, errors: e.errors }); } next(e); } @@ -427,6 +431,15 @@ class InternalServerError extends Error {} class NotImplemented extends Error {} +class AjvError extends Error { + constructor(param: { message: string; errors: ErrorObject[] }) { + super(param.message); + this.errors = param.errors; + } + + errors: ErrorObject[]; +} + export class NcError { static notFound(message = 'Not found') { throw new NotFound(message); @@ -451,4 +464,8 @@ export class NcError { static notImplemented(message = 'Not implemented') { throw new NotImplemented(message); } + + static ajvValidationError(param: { message: string; errors: ErrorObject[] }) { + throw new AjvError(param); + } } From 75af6e0a9bac28f127ceb528c7c56c48f71a5881 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Mar 2023 08:50:01 +0530 Subject: [PATCH 35/64] refactor: move useApis file Signed-off-by: Pranav C --- .../src/lib/controllers/{userApi => userController}/userApis.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/nocodb/src/lib/controllers/{userApi => userController}/userApis.ts (100%) diff --git a/packages/nocodb/src/lib/controllers/userApi/userApis.ts b/packages/nocodb/src/lib/controllers/userController/userApis.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/userApi/userApis.ts rename to packages/nocodb/src/lib/controllers/userController/userApis.ts From 40689609fb5763a8cd250e392b6593cb69d7ef71 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Mar 2023 08:50:37 +0530 Subject: [PATCH 36/64] refactor: update useApis content Signed-off-by: Pranav C --- .../controllers/userController/userApis.ts | 315 +++--------------- 1 file changed, 45 insertions(+), 270 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/userController/userApis.ts b/packages/nocodb/src/lib/controllers/userController/userApis.ts index fa35a5bd0d..574862e25a 100644 --- a/packages/nocodb/src/lib/controllers/userController/userApis.ts +++ b/packages/nocodb/src/lib/controllers/userController/userApis.ts @@ -1,82 +1,24 @@ import { Request, Response } from 'express'; import { TableType, validatePassword } from 'nocodb-sdk'; -import { OrgUserRoles } from 'nocodb-sdk'; -import { NC_APP_SETTINGS } from '../../../constants'; -import Store from '../../../models/Store'; import { T } from 'nc-help'; -import catchError, { NcError } from '../../helpers/catchError'; const { isEmail } = require('validator'); import * as ejs from 'ejs'; import bcrypt from 'bcryptjs'; import { promisify } from 'util'; -import User from '../../../models/User'; const { v4: uuidv4 } = require('uuid'); -import Audit from '../../../models/Audit'; -import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2'; import passport from 'passport'; -import extractProjectIdAndAuthenticate from '../../helpers/extractProjectIdAndAuthenticate'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import { MetaTable } from '../../../utils/globals'; -import Noco from '../../../Noco'; -import { getAjvValidatorMw } from '../helpers'; -import { genJwt } from './helpers'; -import { randomTokenString } from '../../helpers/stringHelpers'; - -export async function registerNewUserIfAllowed({ - firstname, - lastname, - email, - salt, - password, - email_verification_token, -}: { - firstname; - lastname; - email: string; - salt: any; - password; - email_verification_token; -}) { - let roles: string = OrgUserRoles.CREATOR; - - if (await User.isFirst()) { - roles = `${OrgUserRoles.CREATOR},${OrgUserRoles.SUPER_ADMIN}`; - // todo: update in nc_store - // roles = 'owner,creator,editor' - T.emit('evt', { - evt_type: 'project:invite', - count: 1, - }); - } else { - let settings: { invite_only_signup?: boolean } = {}; - try { - settings = JSON.parse((await Store.get(NC_APP_SETTINGS))?.value); - } catch {} - - if (settings?.invite_only_signup) { - NcError.badRequest('Not allowed to signup, contact super admin.'); - } else { - roles = OrgUserRoles.VIEWER; - } - } - - const token_version = randomTokenString(); - - return await User.insert({ - firstname, - lastname, - email, - salt, - password, - email_verification_token, - roles, - token_version, - }); -} +import { getAjvValidatorMw } from '../../meta/api/helpers'; +import catchError, { NcError } from '../../meta/helpers/catchError'; +import extractProjectIdAndAuthenticate from '../../meta/helpers/extractProjectIdAndAuthenticate'; +import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; +import { Audit, User } from '../../models'; +import Noco from '../../Noco'; +import { userService } from '../../services'; export async function signup(req: Request, res: Response) { const { @@ -141,7 +83,7 @@ export async function signup(req: Request, res: Response) { NcError.badRequest('User already exist'); } } else { - await registerNewUserIfAllowed({ + await userService.registerNewUserIfAllowed({ firstname, lastname, email, @@ -171,7 +113,7 @@ export async function signup(req: Request, res: Response) { ); } await promisify((req as any).login.bind(req))(user); - const refreshToken = randomTokenString(); + const refreshToken = userService.randomTokenString(); await User.update(user.id, { refresh_token: refreshToken, email: user.email, @@ -190,18 +132,18 @@ export async function signup(req: Request, res: Response) { }); res.json({ - token: genJwt(user, Noco.getConfig()), + token: userService.genJwt(user, Noco.getConfig()), } as any); } async function successfulSignIn({ - user, - err, - info, - req, - res, - auditDescription, -}) { + user, + err, + info, + req, + res, + auditDescription, + }) { try { if (!user || !user.email) { if (err) { @@ -214,10 +156,10 @@ async function successfulSignIn({ } await promisify((req as any).login.bind(req))(user); - const refreshToken = randomTokenString(); + const refreshToken = userService.randomTokenString(); if (!user.token_version) { - user.token_version = randomTokenString(); + user.token_version = userService.randomTokenString(); } await User.update(user.id, { @@ -236,7 +178,7 @@ async function successfulSignIn({ }); res.json({ - token: genJwt(user, Noco.getConfig()), + token: userService.genJwt(user, Noco.getConfig()), } as any); } catch (e) { console.log(e); @@ -279,15 +221,13 @@ async function googleSignin(req, res, next) { )(req, res, next); } -const REFRESH_TOKEN_COOKIE_KEY = 'refresh_token'; - -function setTokenCookie(res, token): void { +function setTokenCookie(res: Response, token): void { // create http only cookie with refresh token that expires in 7 days const cookieOptions = { httpOnly: true, expires: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000), }; - res.cookie(REFRESH_TOKEN_COOKIE_KEY, token, cookieOptions); + res.cookie('refresh_token', token, cookieOptions); } async function me(req, res): Promise { @@ -298,184 +238,47 @@ async function passwordChange(req: Request, res): Promise { if (!(req as any).isAuthenticated()) { NcError.forbidden('Not allowed'); } - const { currentPassword, newPassword } = req.body; - if (!currentPassword || !newPassword) { - return NcError.badRequest('Missing new/old password'); - } - - // validate password and throw error if password is satisfying the conditions - const { valid, error } = validatePassword(newPassword); - if (!valid) { - NcError.badRequest(`Password : ${error}`); - } - const user = await User.getByEmail((req as any).user.email); - const hashedPassword = await promisify(bcrypt.hash)( - currentPassword, - user.salt - ); - if (hashedPassword !== user.password) { - return NcError.badRequest('Current password is wrong'); - } - - const salt = await promisify(bcrypt.genSalt)(10); - const password = await promisify(bcrypt.hash)(newPassword, salt); - - await User.update(user.id, { - salt, - password, - email: user.email, - token_version: null, - }); - - await Audit.insert({ - op_type: 'AUTHENTICATION', - op_sub_type: 'PASSWORD_CHANGE', - user: user.email, - description: `changed password `, - ip: (req as any).clientIp, + await userService.passwordChange({ + user: req['user'], + req, + body: req.body, }); res.json({ msg: 'Password updated successfully' }); } async function passwordForgot(req: Request, res): Promise { - const _email = req.body.email; - if (!_email) { - NcError.badRequest('Please enter your email address.'); - } - - const email = _email.toLowerCase(); - const user = await User.getByEmail(email); - - if (user) { - const token = uuidv4(); - await User.update(user.id, { - email: user.email, - reset_password_token: token, - reset_password_expires: new Date(Date.now() + 60 * 60 * 1000), - token_version: null, - }); - try { - const template = (await import('./ui/emailTemplates/forgotPassword')) - .default; - await NcPluginMgrv2.emailAdapter().then((adapter) => - adapter.mailSend({ - to: user.email, - subject: 'Password Reset Link', - text: `Visit following link to update your password : ${ - (req as any).ncSiteUrl - }/auth/password/reset/${token}.`, - html: ejs.render(template, { - resetLink: (req as any).ncSiteUrl + `/auth/password/reset/${token}`, - }), - }) - ); - } catch (e) { - console.log(e); - return NcError.badRequest( - 'Email Plugin is not found. Please contact administrators to configure it in App Store first.' - ); - } + await userService.passwordForgot({ + siteUrl: (req as any).ncSiteUrl, + body: req.body, + req, + }); - await Audit.insert({ - op_type: 'AUTHENTICATION', - op_sub_type: 'PASSWORD_FORGOT', - user: user.email, - description: `requested for password reset `, - ip: (req as any).clientIp, - }); - } else { - return NcError.badRequest('Your email has not been registered.'); - } res.json({ msg: 'Please check your email to reset the password' }); } async function tokenValidate(req, res): Promise { - const token = req.params.tokenId; - - const user = await Noco.ncMeta.metaGet(null, null, MetaTable.USERS, { - reset_password_token: token, + await userService.tokenValidate({ + token: req.params.tokenId, }); - - if (!user || !user.email) { - NcError.badRequest('Invalid reset url'); - } - if (new Date(user.reset_password_expires) < new Date()) { - NcError.badRequest('Password reset url expired'); - } res.json(true); } async function passwordReset(req, res): Promise { - const token = req.params.tokenId; - - const user = await Noco.ncMeta.metaGet(null, null, MetaTable.USERS, { - reset_password_token: token, - }); - - if (!user) { - NcError.badRequest('Invalid reset url'); - } - if (user.reset_password_expires < new Date()) { - NcError.badRequest('Password reset url expired'); - } - if (user.provider && user.provider !== 'local') { - NcError.badRequest('Email registered via social account'); - } - - // validate password and throw error if password is satisfying the conditions - const { valid, error } = validatePassword(req.body.password); - if (!valid) { - NcError.badRequest(`Password : ${error}`); - } - - const salt = await promisify(bcrypt.genSalt)(10); - const password = await promisify(bcrypt.hash)(req.body.password, salt); - - await User.update(user.id, { - salt, - password, - email: user.email, - reset_password_expires: null, - reset_password_token: '', - token_version: null, - }); - - await Audit.insert({ - op_type: 'AUTHENTICATION', - op_sub_type: 'PASSWORD_RESET', - user: user.email, - description: `did reset password `, - ip: req.clientIp, + await userService.passwordReset({ + token: req.params.tokenId, + body: req.body, + req, }); res.json({ msg: 'Password reset successful' }); } async function emailVerification(req, res): Promise { - const token = req.params.tokenId; - - const user = await Noco.ncMeta.metaGet(null, null, MetaTable.USERS, { - email_verification_token: token, - }); - - if (!user) { - NcError.badRequest('Invalid verification url'); - } - - await User.update(user.id, { - email: user.email, - email_verification_token: '', - email_verified: true, - }); - - await Audit.insert({ - op_type: 'AUTHENTICATION', - op_sub_type: 'EMAIL_VERIFICATION', - user: user.email, - description: `verified email `, - ip: req.clientIp, + await userService.emailVerification({ + token: req.params.tokenId, + req, }); res.json({ msg: 'Email verified successfully' }); @@ -487,15 +290,13 @@ async function refreshToken(req, res): Promise { return res.status(400).json({ msg: 'Missing refresh token' }); } - const user = await User.getByRefreshToken( - req.cookies[REFRESH_TOKEN_COOKIE_KEY] - ); + const user = await User.getByRefreshToken(req.cookies.refresh_token); if (!user) { return res.status(400).json({ msg: 'Invalid refresh token' }); } - const refreshToken = randomTokenString(); + const refreshToken = userService.randomTokenString(); await User.update(user.id, { email: user.email, @@ -505,7 +306,7 @@ async function refreshToken(req, res): Promise { setTokenCookie(res, refreshToken); res.json({ - token: genJwt(user, Noco.getConfig()), + token: userService.genJwt(user, Noco.getConfig()), } as any); } catch (e) { return res.status(400).json({ msg: e.message }); @@ -526,30 +327,6 @@ async function renderPasswordReset(req, res): Promise { } } -// clear refresh token cookie and update user refresh token to null -const signout = async (req, res): Promise => { - const resBody = { msg: 'Success' }; - if (!req.cookies[REFRESH_TOKEN_COOKIE_KEY]) { - return res.json(resBody); - } - - const user = await User.getByRefreshToken( - req.cookies[REFRESH_TOKEN_COOKIE_KEY] - ); - - if (!user) { - return res.json(resBody); - } - - res.clearCookie(REFRESH_TOKEN_COOKIE_KEY); - - await User.update(user.id, { - refresh_token: null, - }); - - res.json(resBody); -}; - const mapRoutes = (router) => { // todo: old api - /auth/signup?tool=1 router.post( @@ -562,7 +339,6 @@ const mapRoutes = (router) => { getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), catchError(signin) ); - router.post('/auth/user/signout', catchError(signout)); router.get('/auth/user/me', extractProjectIdAndAuthenticate, catchError(me)); router.post( '/auth/password/forgot', @@ -651,7 +427,6 @@ const mapRoutes = (router) => { getAjvValidatorMw('swagger.json#/components/schemas/SignInReq'), catchError(signin) ); - router.post('/api/v1/auth/user/signout', catchError(signout)); router.get( '/api/v1/auth/user/me', extractProjectIdAndAuthenticate, @@ -683,4 +458,4 @@ const mapRoutes = (router) => { // respond with password reset page router.get('/auth/password/reset/:tokenId', catchError(renderPasswordReset)); }; -export { mapRoutes as userApis }; +export { mapRoutes as userController }; From 4e67fd6128b59b84827511c9b5d4bdd82ee4e236 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Mar 2023 09:00:53 +0530 Subject: [PATCH 37/64] fix: corrections Signed-off-by: Pranav C --- packages/nocodb/src/lib/Noco.ts | 2 +- .../src/lib/controllers/dataApis/helpers.ts | 20 +- .../src/lib/controllers/dataApis/index.ts | 2 +- .../dataController/dataAliasNestedApis.ts | 2 +- .../lib/controllers/projectUserController.ts | 2 +- .../controllers/publicApis/publicDataApis.ts | 2 +- .../controllers/swagger/helpers/getPaths.ts | 0 .../controllers/swagger/helpers/getSchemas.ts | 0 .../swagger/helpers/getSwaggerColumnMetas.ts | 0 .../swagger/helpers/getSwaggerJSON.ts | 0 .../swagger/helpers/templates/headers.ts | 0 .../swagger/helpers/templates/params.ts | 0 .../src/lib/controllers/swagger/redocHtml.ts | 93 - .../lib/controllers/swagger/swaggerApis.ts | 61 - .../lib/controllers/swagger/swaggerHtml.ts | 87 - .../lib/controllers/sync/helpers/EntityMap.ts | 222 -- .../sync/helpers/NocoSyncDestAdapter.ts | 6 - .../sync/helpers/NocoSyncSourceAdapter.ts | 7 - .../lib/controllers/sync/helpers/fetchAT.ts | 238 -- .../src/lib/controllers/sync/helpers/job.ts | 2435 ----------------- .../sync/helpers/readAndProcessData.ts | 338 --- .../lib/controllers/sync/helpers/syncMap.ts | 31 - .../src/lib/controllers/sync/importApis.ts | 138 - .../lib/controllers/sync/syncSourceApis.ts | 69 - .../src/lib/controllers/tableController.ts | 12 +- .../src/lib/controllers/userApi/helpers.ts | 23 - .../src/lib/controllers/userApi/index.ts | 1 - .../controllers/userApi/initAdminFromEnv.ts | 283 -- .../lib/controllers/userApi/initStrategies.ts | 336 --- .../userApi/ui/auth/emailVerify.ts | 70 - .../userApi/ui/auth/resetPassword.ts | 108 - .../ui/emailTemplates/forgotPassword.ts | 171 -- .../userApi/ui/emailTemplates/invite.ts | 208 -- .../userApi/ui/emailTemplates/verify.ts | 207 -- packages/nocodb/src/lib/meta/api/index.ts | 12 +- .../src/lib/services/attachmentService.ts | 3 + .../src/lib/services/projectUserService.ts | 2 +- .../nocodb/src/lib/services/tableService.ts | 109 +- .../src/lib/services/userService/helpers.ts | 2 +- 39 files changed, 144 insertions(+), 5158 deletions(-) delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/redocHtml.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/swaggerHtml.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/EntityMap.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncDestAdapter.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncSourceAdapter.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/fetchAT.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/job.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/readAndProcessData.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/syncMap.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/importApis.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/helpers.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/index.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/initStrategies.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/ui/auth/emailVerify.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/ui/auth/resetPassword.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/forgotPassword.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/invite.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/verify.ts diff --git a/packages/nocodb/src/lib/Noco.ts b/packages/nocodb/src/lib/Noco.ts index ca4ae28486..151f211c65 100644 --- a/packages/nocodb/src/lib/Noco.ts +++ b/packages/nocodb/src/lib/Noco.ts @@ -45,7 +45,7 @@ import User from './models/User'; import * as http from 'http'; import weAreHiring from './utils/weAreHiring'; import getInstance from './utils/getInstance'; -import initAdminFromEnv from './controllers/userApi/initAdminFromEnv'; +import initAdminFromEnv from './services/userService/initAdminFromEnv'; const log = debug('nc:app'); require('dotenv').config(); diff --git a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts index 1a7216f5fc..2ca5cd17e1 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts @@ -1,19 +1,19 @@ -import Project from '../../../models/Project'; -import Model from '../../../models/Model'; -import View from '../../../models/View'; -import { NcError } from '../../helpers/catchError'; +import { NcError } from '../../meta/helpers/catchError'; +import Project from '../../models/Project'; +import Model from '../../models/Model'; +import View from '../../models/View'; import { Request } from 'express'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; import { isSystemColumn, UITypes } from 'nocodb-sdk'; import * as XLSX from 'xlsx'; -import Column from '../../../models/Column'; -import LookupColumn from '../../../models/LookupColumn'; -import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; +import Column from '../../models/Column'; +import LookupColumn from '../../models/LookupColumn'; +import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; import papaparse from 'papaparse'; -import { dataService } from '../../../services'; +import { dataService } from '../../services'; export async function getViewAndModelFromRequestByAliasOrId( req: | Request<{ projectName: string; tableName: string; viewName?: string }> diff --git a/packages/nocodb/src/lib/controllers/dataApis/index.ts b/packages/nocodb/src/lib/controllers/dataApis/index.ts index 3a4528a5e1..f7b699f340 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/index.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/index.ts @@ -2,7 +2,7 @@ import dataApis from './dataApis'; import oldDataApis from './oldDataApis'; import dataAliasApis from './dataAliasApis'; import bulkDataAliasApis from './bulkDataAliasApis'; -import dataAliasNestedApis from '../../../controllers/dataController/dataAliasNestedApis'; +import dataAliasNestedApis from '../../controllers/dataController/dataAliasNestedApis'; import dataAliasExportApis from './dataAliasExportApis'; export { diff --git a/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts b/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts index eb11f56d45..d0dc296834 100644 --- a/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts +++ b/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts @@ -7,7 +7,7 @@ import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; import { getColumnByIdOrName, getViewAndModelFromRequestByAliasOrId, -} from '../../meta/api/dataApis/helpers'; +} from '../dataApis/helpers'; import { NcError } from '../../meta/helpers/catchError'; import apiMetrics from '../../meta/helpers/apiMetrics'; diff --git a/packages/nocodb/src/lib/controllers/projectUserController.ts b/packages/nocodb/src/lib/controllers/projectUserController.ts index 270db703ed..6be2ff985c 100644 --- a/packages/nocodb/src/lib/controllers/projectUserController.ts +++ b/packages/nocodb/src/lib/controllers/projectUserController.ts @@ -270,7 +270,7 @@ export async function sendInviteEmail( req: any ): Promise { try { - const template = (await import('./userApi/ui/emailTemplates/invite')) + const template = (await import('./userController/ui/emailTemplates/invite')) .default; const emailAdapter = await NcPluginMgrv2.emailAdapter(); diff --git a/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts b/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts index 17e58cf1a5..792a6b4b61 100644 --- a/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts +++ b/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts @@ -15,7 +15,7 @@ import path from 'path'; import { nanoid } from 'nanoid'; import { mimeIcons } from '../../utils/mimeTypes'; import slash from 'slash'; -import { sanitizeUrlPath } from '../attachmentController'; +import { sanitizeUrlPath } from '../../services/attachmentService'; import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; import { getColumnByIdOrName } from '../dataApis/helpers'; import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants'; diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/redocHtml.ts b/packages/nocodb/src/lib/controllers/swagger/redocHtml.ts deleted file mode 100644 index cc2682fdac..0000000000 --- a/packages/nocodb/src/lib/controllers/swagger/redocHtml.ts +++ /dev/null @@ -1,93 +0,0 @@ -export default ({ - ncSiteUrl, -}: { - ncSiteUrl: string; -}): string => ` - - - NocoDB API Documentation - - - - - - - - -
- - - - -`; diff --git a/packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts b/packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts deleted file mode 100644 index ef44542646..0000000000 --- a/packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts +++ /dev/null @@ -1,61 +0,0 @@ -// @ts-ignore -import catchError, { NcError } from '../../meta/helpers/catchError'; -import { Router } from 'express'; -import Model from '../../models/Model'; -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import getSwaggerJSON from './helpers/getSwaggerJSON'; -import Project from '../../models/Project'; -import getSwaggerHtml from './swaggerHtml'; -import getRedocHtml from './redocHtml'; - -async function swaggerJson(req, res) { - const project = await Project.get(req.params.projectId); - - if (!project) NcError.notFound(); - - const models = await Model.list({ - project_id: req.params.projectId, - base_id: null, - }); - - const swagger = await getSwaggerJSON(project, models); - - swagger.servers = [ - { - url: req.ncSiteUrl, - }, - { - url: '{customUrl}', - variables: { - customUrl: { - default: req.ncSiteUrl, - description: 'Provide custom nocodb app base url', - }, - }, - }, - ] as any; - - res.json(swagger); -} - -function swaggerHtml(_, res) { - res.send(getSwaggerHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' })); -} - -function redocHtml(_, res) { - res.send(getRedocHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' })); -} - -const router = Router({ mergeParams: true }); - -// todo: auth -router.get( - '/api/v1/db/meta/projects/:projectId/swagger.json', - ncMetaAclMw(swaggerJson, 'swaggerJson') -); - -router.get('/api/v1/db/meta/projects/:projectId/swagger', swaggerHtml); - -router.get('/api/v1/db/meta/projects/:projectId/redoc', redocHtml); - -export default router; diff --git a/packages/nocodb/src/lib/controllers/swagger/swaggerHtml.ts b/packages/nocodb/src/lib/controllers/swagger/swaggerHtml.ts deleted file mode 100644 index 27f6bef73d..0000000000 --- a/packages/nocodb/src/lib/controllers/swagger/swaggerHtml.ts +++ /dev/null @@ -1,87 +0,0 @@ -export default ({ - ncSiteUrl, -}: { - ncSiteUrl: string; -}): string => ` - - - NocoDB : API Docs - - - - - - -
- - - -`; diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/EntityMap.ts b/packages/nocodb/src/lib/controllers/sync/helpers/EntityMap.ts deleted file mode 100644 index bd0bdd86ca..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/EntityMap.ts +++ /dev/null @@ -1,222 +0,0 @@ -import sqlite3 from 'sqlite3'; -import { Readable } from 'stream'; - -class EntityMap { - initialized: boolean; - cols: string[]; - db: any; - - constructor(...args) { - this.initialized = false; - this.cols = args.map((arg) => processKey(arg)); - this.db = new Promise((resolve, reject) => { - const db = new sqlite3.Database(':memory:'); - - const colStatement = - this.cols.length > 0 - ? this.cols.join(' TEXT, ') + ' TEXT' - : 'mappingPlaceholder TEXT'; - db.run(`CREATE TABLE mapping (${colStatement})`, (err) => { - if (err) { - console.log(err); - reject(err); - } - resolve(db); - }); - }); - } - - async init() { - if (!this.initialized) { - this.db = await this.db; - this.initialized = true; - } - } - - destroy() { - if (this.initialized && this.db) { - this.db.close(); - } - } - - async addRow(row) { - if (!this.initialized) { - throw 'Please initialize first!'; - } - - const cols = Object.keys(row).map((key) => processKey(key)); - const colStatement = cols.map((key) => `'${key}'`).join(', '); - const questionMarks = cols.map(() => '?').join(', '); - - const promises = []; - - for (const col of cols.filter((col) => !this.cols.includes(col))) { - promises.push( - new Promise((resolve, reject) => { - this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => { - if (err) { - console.log(err); - reject(err); - } - this.cols.push(col); - resolve(true); - }); - }) - ); - } - - await Promise.all(promises); - - const values = Object.values(row).map((val) => { - if (typeof val === 'object') { - return `JSON::${JSON.stringify(val)}`; - } - return val; - }); - - return new Promise((resolve, reject) => { - this.db.run( - `INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`, - values, - (err) => { - if (err) { - console.log(err); - reject(err); - } - resolve(true); - } - ); - }); - } - - getRow(col, val, res = []): Promise> { - if (!this.initialized) { - throw 'Please initialize first!'; - } - return new Promise((resolve, reject) => { - col = processKey(col); - res = res.map((r) => processKey(r)); - this.db.get( - `SELECT ${ - res.length ? res.join(', ') : '*' - } FROM mapping WHERE ${col} = ?`, - [val], - (err, rs) => { - if (err) { - console.log(err); - reject(err); - } - if (rs) { - rs = processResponseRow(rs); - } - resolve(rs); - } - ); - }); - } - - getCount(): Promise { - if (!this.initialized) { - throw 'Please initialize first!'; - } - return new Promise((resolve, reject) => { - this.db.get(`SELECT COUNT(*) as count FROM mapping`, (err, rs) => { - if (err) { - console.log(err); - reject(err); - } - resolve(rs.count); - }); - }); - } - - getStream(res = []): DBStream { - if (!this.initialized) { - throw 'Please initialize first!'; - } - res = res.map((r) => processKey(r)); - return new DBStream( - this.db, - `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping` - ); - } - - getLimit(limit, offset, res = []): Promise[]> { - if (!this.initialized) { - throw 'Please initialize first!'; - } - return new Promise((resolve, reject) => { - res = res.map((r) => processKey(r)); - this.db.all( - `SELECT ${ - res.length ? res.join(', ') : '*' - } FROM mapping LIMIT ${limit} OFFSET ${offset}`, - (err, rs) => { - if (err) { - console.log(err); - reject(err); - } - for (let row of rs) { - row = processResponseRow(row); - } - resolve(rs); - } - ); - }); - } -} - -class DBStream extends Readable { - db: any; - stmt: any; - sql: any; - - constructor(db, sql) { - super({ objectMode: true }); - this.db = db; - this.sql = sql; - this.stmt = this.db.prepare(this.sql); - this.on('end', () => this.stmt.finalize()); - } - - _read() { - let stream = this; - this.stmt.get(function (err, result) { - if (err) { - stream.emit('error', err); - } else { - if (result) { - result = processResponseRow(result); - } - stream.push(result || null); - } - }); - } -} - -function processResponseRow(res: any) { - for (const key of Object.keys(res)) { - if (res[key] && res[key].startsWith('JSON::')) { - try { - res[key] = JSON.parse(res[key].replace('JSON::', '')); - } catch (e) { - console.log(e); - } - } - if (revertKey(key) !== key) { - res[revertKey(key)] = res[key]; - delete res[key]; - } - } - return res; -} - -function processKey(key) { - return key.replace(/'/g, "''").replace(/[A-Z]/g, (match) => `_${match}`); -} - -function revertKey(key) { - return key.replace(/''/g, "'").replace(/_[A-Z]/g, (match) => match[1]); -} - -export default EntityMap; diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncDestAdapter.ts b/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncDestAdapter.ts deleted file mode 100644 index 8af4a493c5..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncDestAdapter.ts +++ /dev/null @@ -1,6 +0,0 @@ -export abstract class NocoSyncSourceAdapter { - public abstract init(): Promise; - public abstract destProjectWrite(): Promise; - public abstract destSchemaWrite(): Promise; - public abstract destDataWrite(): Promise; -} diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncSourceAdapter.ts b/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncSourceAdapter.ts deleted file mode 100644 index a419d1c24d..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncSourceAdapter.ts +++ /dev/null @@ -1,7 +0,0 @@ -export abstract class NocoSyncSourceAdapter { - public abstract init(): Promise; - public abstract srcSchemaGet(): Promise; - public abstract srcDataLoad(): Promise; - public abstract srcDataListen(): Promise; - public abstract srcDataPoll(): Promise; -} diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/fetchAT.ts b/packages/nocodb/src/lib/controllers/sync/helpers/fetchAT.ts deleted file mode 100644 index 4ada40bd17..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/fetchAT.ts +++ /dev/null @@ -1,238 +0,0 @@ -const axios = require('axios').default; - -const info: any = { - initialized: false, -}; - -async function initialize(shareId) { - info.cookie = ''; - const url = `https://airtable.com/${shareId}`; - - try { - const hreq = await axios - .get(url, { - headers: { - accept: - 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'document', - 'sec-fetch-mode': 'navigate', - 'sec-fetch-site': 'none', - 'sec-fetch-user': '?1', - 'upgrade-insecure-requests': '1', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - }, - referrerPolicy: 'strict-origin-when-cross-origin', - body: null, - method: 'GET', - }) - .then((response) => { - for (const ck of response.headers['set-cookie']) { - info.cookie += ck.split(';')[0] + '; '; - } - return response.data; - }) - .catch(() => { - throw { - message: - 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - }); - - info.headers = JSON.parse( - hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim() - ); - info.link = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim()); - info.baseInfo = decodeURIComponent(info.link) - .match(/{(.*)}/g)[0] - .split('&') - .reduce((result, el) => { - try { - return Object.assign( - result, - JSON.parse(el.includes('=') ? el.split('=')[1] : el) - ); - } catch (e) { - if (el.includes('=')) { - return Object.assign(result, { - [el.split('=')[0]]: el.split('=')[1], - }); - } - } - }, {}); - info.baseId = info.baseInfo.applicationId; - info.initialized = true; - } catch (e) { - console.log(e); - info.initialized = false; - if (e.message) { - throw e; - } else { - throw { - message: - 'Error processing Shared Base :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - } - } -} - -async function read() { - if (info.initialized) { - const resreq = await axios('https://airtable.com' + info.link, { - headers: { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'x-time-zone': 'Europe/Berlin', - cookie: info.cookie, - ...info.headers, - }, - referrerPolicy: 'no-referrer', - body: null, - method: 'GET', - }) - .then((response) => { - return response.data; - }) - .catch(() => { - throw { - message: - 'Error Reading :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - }); - - return { - schema: resreq.data, - baseId: info.baseId, - baseInfo: info.baseInfo, - }; - } else { - throw { - message: 'Error Initializing :: please try again !!', - }; - } -} - -async function readView(viewId) { - if (info.initialized) { - const resreq = await axios( - `https://airtable.com/v0.3/view/${viewId}/readData?` + - `stringifiedObjectParams=${encodeURIComponent('{}')}&requestId=${ - info.baseInfo.requestId - }&accessPolicy=${encodeURIComponent( - JSON.stringify({ - allowedActions: info.baseInfo.allowedActions, - shareId: info.baseInfo.shareId, - applicationId: info.baseInfo.applicationId, - generationNumber: info.baseInfo.generationNumber, - expires: info.baseInfo.expires, - signature: info.baseInfo.signature, - }) - )}`, - { - headers: { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'x-time-zone': 'Europe/Berlin', - cookie: info.cookie, - ...info.headers, - }, - referrerPolicy: 'no-referrer', - body: null, - method: 'GET', - } - ) - .then((response) => { - return response.data; - }) - .catch(() => { - throw { - message: - 'Error Reading View :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - }); - return { view: resreq.data }; - } else { - throw { - message: 'Error Initializing :: please try again !!', - }; - } -} - -async function readTemplate(templateId) { - if (!info.initialized) { - await initialize('shrO8aYf3ybwSdDKn'); - } - const resreq = await axios( - `https://www.airtable.com/v0.3/exploreApplications/${templateId}`, - { - headers: { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'x-time-zone': 'Europe/Berlin', - cookie: info.cookie, - ...info.headers, - }, - referrer: 'https://www.airtable.com/', - referrerPolicy: 'same-origin', - body: null, - method: 'GET', - mode: 'cors', - credentials: 'include', - } - ) - .then((response) => { - return response.data; - }) - .catch(() => { - throw { - message: - 'Error Fetching :: Ensure www.airtable.com/templates/featured/ is accessible.', - }; - }); - return { template: resreq }; -} - -function unicodeToChar(text) { - return text.replace(/\\u[\dA-F]{4}/gi, function (match) { - return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16)); - }); -} - -export default { - initialize, - read, - readView, - readTemplate, -}; diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/job.ts b/packages/nocodb/src/lib/controllers/sync/helpers/job.ts deleted file mode 100644 index 9e55beb4bd..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/job.ts +++ /dev/null @@ -1,2435 +0,0 @@ -import { T } from 'nc-help'; -import FetchAT from './fetchAT'; -import { UITypes } from 'nocodb-sdk'; -// import * as sMap from './syncMap'; - -import { Api } from 'nocodb-sdk'; - -import Airtable from 'airtable'; -import jsonfile from 'jsonfile'; -import hash from 'object-hash'; -import { promisify } from 'util'; - -import dayjs from 'dayjs'; -import tinycolor from 'tinycolor2'; -import { importData, importLTARData } from './readAndProcessData'; - -import EntityMap from './EntityMap'; - -const writeJsonFileAsync = promisify(jsonfile.writeFile); - -const selectColors = { - // normal - blue: '#cfdfff', - cyan: '#d0f0fd', - teal: '#c2f5e9', - green: '#d1f7c4', - orange: '#fee2d5', - yellow: '#ffeab6', - red: '#ffdce5', - pink: '#ffdaf6', - purple: '#ede2fe', - gray: '#eee', - // medium - blueMedium: '#9cc7ff', - cyanMedium: '#77d1f3', - tealMedium: '#72ddc3', - greenMedium: '#93e088', - orangeMedium: '#ffa981', - yellowMedium: '#ffd66e', - redMedium: '#ff9eb7', - pinkMedium: '#f99de2', - purpleMedium: '#cdb0ff', - grayMedium: '#ccc', - // dark - blueDark: '#2d7ff9', - cyanDark: '#18bfff', - tealDark: '#20d9d2', - greenDark: '#20c933', - orangeDark: '#ff6f2c', - yellowDark: '#fcb400', - redDark: '#f82b60', - pinkDark: '#ff08c2', - purpleDark: '#8b46ff', - grayDark: '#666', - // darker - blueDarker: '#2750ae', - cyanDarker: '#0b76b7', - tealDarker: '#06a09b', - greenDarker: '#338a17', - orangeDarker: '#d74d26', - yellowDarker: '#b87503', - redDarker: '#ba1e45', - pinkDarker: '#b2158b', - purpleDarker: '#6b1cb0', - grayDarker: '#444', -}; - -export default async ( - syncDB: AirtableSyncConfig, - progress: (data: { msg?: string; level?: any }) => void -) => { - const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent'); - await sMapEM.init(); - - const sMap = { - // static mapping records between aTblId && ncId - async addToMappingTbl(aTblId, ncId, ncName, ncParent?) { - await sMapEM.addRow({ aTblId, ncId, ncName, ncParent }); - }, - - // get NcID from airtable ID - async getNcIdFromAtId(aId) { - return (await sMapEM.getRow('aTblId', aId, ['ncId']))?.ncId; - }, - - // get nc Parent from airtable ID - async getNcParentFromAtId(aId) { - return (await sMapEM.getRow('aTblId', aId, ['ncParent']))?.ncParent; - }, - - // get nc-title from airtable ID - async getNcNameFromAtId(aId) { - return (await sMapEM.getRow('aTblId', aId, ['ncName']))?.ncName; - }, - }; - - function logBasic(log) { - progress({ level: 0, msg: log }); - } - - function logDetailed(log) { - if (debugMode) progress({ level: 1, msg: log }); - } - - const perfStats = []; - function recordPerfStart() { - if (!debugMode) return 0; - return Date.now(); - } - function recordPerfStats(start, event) { - if (!debugMode) return; - const duration = Date.now() - start; - perfStats.push({ d: duration, e: event }); - } - - let base, baseId; - const start = Date.now(); - const enableErrorLogs = false; - const generate_migrationStats = true; - const debugMode = false; - let api: Api; - let g_aTblSchema = []; - let ncCreatedProjectSchema: any = {}; - const ncLinkMappingTable: any[] = []; - const nestedLookupTbl: any[] = []; - const nestedRollupTbl: any[] = []; - const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' }; - const storeLinks = false; - const ncLinkDataStore: any = {}; - const insertedAssocRef: any = {}; - - const atNcAliasRef: { - [ncTableId: string]: { - [ncTitle: string]: string; - }; - } = {}; - - const uniqueTableNameGen = getUniqueNameGenerator('sheet'); - - // run time counter (statistics) - const rtc = { - sort: 0, - filter: 0, - view: { - total: 0, - grid: 0, - gallery: 0, - form: 0, - }, - fetchAt: { - count: 0, - time: 0, - }, - migrationSkipLog: { - count: 0, - log: [], - }, - data: { - records: 0, - nestedLinks: 0, - }, - }; - - function updateMigrationSkipLog(tbl, col, type, reason?) { - rtc.migrationSkipLog.count++; - rtc.migrationSkipLog.log.push( - `tn[${tbl}] cn[${col}] type[${type}] :: ${reason}` - ); - } - - // mapping table - // - - async function getAirtableSchema(sDB) { - const start = Date.now(); - - if (!sDB.shareId) - throw { - message: - 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - - if (sDB.shareId.startsWith('exp')) { - const template = await FetchAT.readTemplate(sDB.shareId); - await FetchAT.initialize(template.template.exploreApplication.shareId); - } else { - await FetchAT.initialize(sDB.shareId); - } - const ft = await FetchAT.read(); - const duration = Date.now() - start; - rtc.fetchAt.count++; - rtc.fetchAt.time += duration; - - if (!ft.baseId) { - throw { - message: - 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - } - - const file = ft.schema; - baseId = ft.baseId; - base = new Airtable({ apiKey: sDB.apiKey }).base(baseId); - // store copy of airtable schema globally - g_aTblSchema = file.tableSchemas; - - if (debugMode) - await writeJsonFileAsync('aTblSchema.json', ft, { spaces: 2 }); - - return file; - } - - async function getViewData(viewId) { - const start = Date.now(); - const ft = await FetchAT.readView(viewId); - const duration = Date.now() - start; - rtc.fetchAt.count++; - rtc.fetchAt.time += duration; - - if (debugMode) - await writeJsonFileAsync(`${viewId}.json`, ft, { spaces: 2 }); - return ft.view; - } - - function getRootDbType() { - return ncCreatedProjectSchema?.bases.find((el) => el.id === syncDB.baseId) - ?.type; - } - - // base mapping table - const aTblNcTypeMap = { - foreignKey: UITypes.LinkToAnotherRecord, - text: UITypes.SingleLineText, - multilineText: UITypes.LongText, - richText: UITypes.LongText, - multipleAttachment: UITypes.Attachment, - checkbox: UITypes.Checkbox, - multiSelect: UITypes.MultiSelect, - select: UITypes.SingleSelect, - collaborator: UITypes.Collaborator, - multiCollaborator: UITypes.Collaborator, - date: UITypes.Date, - phone: UITypes.PhoneNumber, - number: UITypes.Decimal, - rating: UITypes.Rating, - formula: UITypes.Formula, - rollup: UITypes.Rollup, - count: UITypes.Count, - lookup: UITypes.Lookup, - autoNumber: UITypes.AutoNumber, - barcode: UITypes.SingleLineText, - button: UITypes.Button, - }; - - //----------------------------------------------------------------------------- - // aTbl helper routines - // - - function nc_sanitizeName(name) { - // replace all special characters by _ - return name.replace(/\W+/g, '_').trim(); - } - - function nc_getSanitizedColumnName(table, name) { - let col_name = nc_sanitizeName(name); - - // truncate to 60 chars if character if exceeds above 60 - col_name = col_name?.slice(0, 60); - - // for knex, replace . with _ - const col_alias = name.trim().replace(/\./g, '_'); - - // check if already a column exists with same name? - const duplicateTitle = table.columns.find( - (x) => x.title?.toLowerCase() === col_alias?.toLowerCase() - ); - const duplicateColumn = table.columns.find( - (x) => x.column_name?.toLowerCase() === col_name?.toLowerCase() - ); - if (duplicateTitle) { - if (enableErrorLogs) console.log(`## Duplicate title ${col_alias}`); - } - - return { - // kludge: error observed in Nc with space around column-name - title: col_alias + (duplicateTitle ? '_2' : ''), - column_name: col_name + (duplicateColumn ? '_2' : ''), - }; - } - - // aTbl: retrieve table name from table ID - // - // @ts-ignore - function aTbl_getTableName(tblId) { - const sheetObj = g_aTblSchema.find((tbl) => tbl.id === tblId); - return { - tn: sheetObj.name, - }; - } - - const ncSchema = { - tables: [], - tablesById: {}, - }; - - // aTbl: retrieve column name from column ID - // - function aTbl_getColumnName(colId): any { - for (let i = 0; i < g_aTblSchema.length; i++) { - const sheetObj = g_aTblSchema[i]; - const column = sheetObj.columns.find((col) => col.id === colId); - if (column !== undefined) - return { - tn: sheetObj.name, - cn: column.name, - }; - } - } - - // nc dump schema - // - // @ts-ignore - async function nc_DumpTableSchema() { - console.log('['); - const ncTblList = await api.base.tableList( - ncCreatedProjectSchema.id, - syncDB.baseId - ); - for (let i = 0; i < ncTblList.list.length; i++) { - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); - console.log(JSON.stringify(ncTbl, null, 2)); - console.log(','); - } - console.log(']'); - } - - // retrieve nc column schema from using aTbl field ID as reference - // - async function nc_getColumnSchema(aTblFieldId) { - // let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); - // let aTblField = aTbl_getColumnName(aTblFieldId); - // let ncTblId = ncTblList.list.filter(x => x.title === aTblField.tn)[0].id; - // let ncTbl = await api.dbTable.read(ncTblId); - // let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn); - // return ncCol; - - const ncTblId = await sMap.getNcParentFromAtId(aTblFieldId); - const ncColId = await sMap.getNcIdFromAtId(aTblFieldId); - - // not migrated column, skip - if (ncColId === undefined || ncTblId === undefined) return 0; - - return ncSchema.tablesById[ncTblId].columns.find((x) => x.id === ncColId); - } - - // retrieve nc table schema using table name - // optimize: create a look-up table & re-use information - // - async function nc_getTableSchema(tableName) { - // let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); - // let ncTblId = ncTblList.list.filter(x => x.title === tableName)[0].id; - // let ncTbl = await api.dbTable.read(ncTblId); - // return ncTbl; - - return ncSchema.tables.find((x) => x.title === tableName); - } - - // delete project if already exists - async function init({ - projectName, - }: { - projectName?: string; - projectId?: string; - }) { - // delete 'sample' project if already exists - const x = await api.project.list(); - - const sampleProj = x.list.find((a) => a.title === projectName); - if (sampleProj) { - await api.project.delete(sampleProj.id); - } - logDetailed('Init'); - } - - // map UIDT - // - function getNocoType(col) { - // start with default map - let ncType = aTblNcTypeMap[col.type]; - - // types email & url are marked as text - // types currency & percent, duration are marked as number - // types createTime & modifiedTime are marked as formula - - switch (col.type) { - case 'text': - if (col.typeOptions?.validatorName === 'email') ncType = UITypes.Email; - else if (col.typeOptions?.validatorName === 'url') ncType = UITypes.URL; - break; - - case 'number': - // kludge: currency validation error with decimal places - if (col.typeOptions?.format === 'percentV2') ncType = UITypes.Percent; - else if (col.typeOptions?.format === 'duration') - ncType = UITypes.Duration; - else if (col.typeOptions?.format === 'currency') - ncType = UITypes.Currency; - else if (col.typeOptions?.precision > 0) ncType = UITypes.Decimal; - break; - - case 'formula': - if (col.typeOptions?.formulaTextParsed === 'CREATED_TIME()') - ncType = UITypes.DateTime; - else if (col.typeOptions?.formulaTextParsed === 'LAST_MODIFIED_TIME()') - ncType = UITypes.DateTime; - break; - - case 'computation': - if (col.typeOptions?.resultType === 'collaborator') - ncType = UITypes.Collaborator; - break; - - case 'date': - if (col.typeOptions?.isDateTime) ncType = UITypes.DateTime; - break; - - // case 'barcode': - // case 'button': - // ncType = UITypes.SingleLineText; - // break; - } - - return ncType; - } - - // retrieve additional options associated with selected data types - // - async function getNocoTypeOptions(col: any): Promise { - switch (col.type) { - case 'select': - case 'multiSelect': { - // prepare options list in CSV format - // note: NC doesn't allow comma's in options - // - const options = []; - let order = 1; - for (const [, value] of Object.entries(col.typeOptions.choices)) { - // replace commas with dot for multiselect - if (col.type === 'multiSelect') { - (value as any).name = (value as any).name.replace(/,/g, '.'); - } - // we don't allow empty records, placeholder instead - if ((value as any).name === '') { - (value as any).name = 'nc_empty'; - } - // enumerate duplicates (we don't allow them) - // TODO fix record mapping (this causes every record to map first option, we can't handle them using data api as they don't provide option id within data we might instead get the correct mapping from schema file ) - let dupNo = 1; - const defaultName = (value as any).name; - while ( - options.find( - (el) => - el.title.toLowerCase() === (value as any).name.toLowerCase() - ) - ) { - (value as any).name = `${defaultName}_${dupNo++}`; - } - options.push({ - order: order++, - title: (value as any).name, - color: selectColors[(value as any).color] - ? selectColors[(value as any).color] - : tinycolor.random().toHexString(), - }); - - await sMap.addToMappingTbl( - (value as any).id, - undefined, - (value as any).name - ); - } - return { type: col.type, data: options }; - } - default: - return { type: undefined }; - } - } - - // convert to Nc schema (basic, excluding relations) - // - async function tablesPrepare(tblSchema: any[]) { - const tables: any[] = []; - - for (let i = 0; i < tblSchema.length; ++i) { - const table: any = {}; - - if (syncDB.options.syncViews) { - rtc.view.total += tblSchema[i].views.reduce( - (acc, cur) => - ['grid', 'form', 'gallery'].includes(cur.type) ? ++acc : acc, - 0 - ); - } else { - rtc.view.total = tblSchema.length; - } - - // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id; - table.title = tblSchema[i].name; - let sanitizedName = nc_sanitizeName(tblSchema[i].name); - - // truncate to 50 chars if character if exceeds above 50 - // upto 64 should be fine but we are keeping it to 50 since - // meta project adds prefix as well - sanitizedName = sanitizedName?.slice(0, 50); - - // check for duplicate and populate a unique name if already exist - table.table_name = uniqueTableNameGen(sanitizedName); - - const uniqueColNameGen = getUniqueNameGenerator('field'); - table.columns = []; - const sysColumns = [ - { - title: ncSysFields.id, - column_name: ncSysFields.id, - uidt: UITypes.ID, - meta: { - ag: 'nc', - }, - }, - { - title: ncSysFields.hash, - column_name: ncSysFields.hash, - uidt: UITypes.SingleLineText, - system: true, - }, - ]; - - for (let j = 0; j < tblSchema[i].columns.length; j++) { - const col = tblSchema[i].columns[j]; - - // skip link, lookup, rollup fields in this iteration - if (['foreignKey', 'lookup', 'rollup'].includes(col.type)) { - continue; - } - - // base column schema - const ncName: any = nc_getSanitizedColumnName(table, col.name); - const ncCol: any = { - // Enable to use aTbl identifiers as is: id: col.id, - title: ncName.title, - column_name: uniqueColNameGen(ncName.column_name), - uidt: getNocoType(col), - }; - - // not supported datatype: pure formula field - // allow formula based computed fields (created time/ modified time to go through) - if (ncCol.uidt === UITypes.Formula) { - updateMigrationSkipLog( - tblSchema[i].name, - ncName.title, - col.type, - 'column type not supported' - ); - continue; - } - - // populate cdf (column default value) if configured - // if (col?.default) { - // if (typeof col.default === 'string') - // ncCol.cdf = `'${col.default.replace?.(/'/g, "\\'")}'`; - // else ncCol.cdf = col.default; - // } - - // change from default 'tinytext' as airtable allows more than 255 characters - // for single line text column type - if (col.type === 'text') ncCol.dt = 'text'; - - // #fix-2363-decimal-out-of-range - if (['sqlite3', 'mysql2'].includes(getRootDbType())) { - if (ncCol.uidt === UITypes.Decimal) { - ncCol.dt = 'double'; - ncCol.dtxp = 22; - ncCol.dtxs = '2'; - } - } - - // additional column parameters when applicable - const colOptions = await getNocoTypeOptions(col); - - switch (colOptions.type) { - case 'select': - case 'multiSelect': - ncCol.colOptions = { - options: [...colOptions.data], - }; - - if (['mysql', 'mysql2'].includes(getRootDbType())) { - // if options are empty, configure '' as an option - ncCol.dtxp = - colOptions.data - .map((el) => `'${el.title.replace(/'/gi, "''")}'`) - .join(',') || "''"; - } - - break; - case undefined: - break; - } - table.columns.push(ncCol); - } - table.columns.push(sysColumns[0]); - table.columns.push(sysColumns[1]); - - tables.push(table); - } - return tables; - } - - async function nocoCreateBaseSchema(aTblSchema) { - // base schema preparation: exclude - const tables: any[] = await tablesPrepare(aTblSchema); - - // for each table schema, create nc table - for (let idx = 0; idx < tables.length; idx++) { - logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`); - - logDetailed(`NC API: base.tableCreate ${tables[idx].title}`); - - let _perfStart = recordPerfStart(); - const table: any = await api.base.tableCreate( - ncCreatedProjectSchema.id, - syncDB.baseId, - tables[idx] - ); - recordPerfStats(_perfStart, 'dbTable.create'); - - updateNcTblSchema(table); - - // update mapping table - await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title); - for (let colIdx = 0; colIdx < table.columns.length; colIdx++) { - const aId = aTblSchema[idx].columns.find( - (x) => - x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title - )?.id; - if (aId) - await sMap.addToMappingTbl( - aId, - table.columns[colIdx].id, - table.columns[colIdx].title, - table.id - ); - } - - // update default view name- to match it to airtable view name - logDetailed(`NC API: dbView.list ${table.id}`); - _perfStart = recordPerfStart(); - const view = await api.dbView.list(table.id); - recordPerfStats(_perfStart, 'dbView.list'); - - const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid'); - logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); - _perfStart = recordPerfStart(); - await api.dbView.update(view.list[0].id, { - title: aTbl_grid.name, - }); - recordPerfStats(_perfStart, 'dbView.update'); - - await updateNcTblSchemaById(table.id); - - await sMap.addToMappingTbl( - aTbl_grid.id, - table.views[0].id, - aTbl_grid.name, - table.id - ); - } - - return tables; - } - - async function nocoCreateLinkToAnotherRecord(aTblSchema) { - // Link to another RECORD - for (let idx = 0; idx < aTblSchema.length; idx++) { - const aTblLinkColumns = aTblSchema[idx].columns.filter( - (x) => x.type === 'foreignKey' - ); - - // Link columns exist - // - if (aTblLinkColumns.length) { - for (let i = 0; i < aTblLinkColumns.length; i++) { - logDetailed( - `[${idx + 1}/${aTblSchema.length}] Configuring Links :: [${i + 1}/${ - aTblLinkColumns.length - }] ${aTblSchema[idx].name}` - ); - - // for self links, there is no symmetric column - { - const src = aTbl_getColumnName(aTblLinkColumns[i].id); - const dst = aTbl_getColumnName( - aTblLinkColumns[i].typeOptions?.symmetricColumnId - ); - logDetailed( - `LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst?.tn}:${dst?.cn}` - ); - } - - // check if link already established? - if (!nc_isLinkExists(aTblLinkColumns[i].id)) { - // parent table ID - // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; - const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - - // find child table name from symmetric column ID specified - // self link, symmetricColumnId field will be undefined - const childTable = aTbl_getColumnName( - aTblLinkColumns[i].typeOptions?.symmetricColumnId - ); - - // retrieve child table ID (nc) from table name - let childTableId = srcTableId; - if (childTable) { - childTableId = (await nc_getTableSchema(childTable.tn)).id; - } - - // check if already a column exists with this name? - let _perfStart = recordPerfStart(); - const srcTbl: any = await api.dbTable.read(srcTableId); - recordPerfStats(_perfStart, 'dbTable.read'); - - // create link - const ncName = nc_getSanitizedColumnName( - srcTbl, - aTblLinkColumns[i].name - ); - - // LTAR alias ref to AT - atNcAliasRef[srcTbl.id] = atNcAliasRef[srcTbl.id] || {}; - atNcAliasRef[srcTbl.id][ncName.title] = aTblLinkColumns[i].name; - - logDetailed( - `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` - ); - _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.LinkToAnotherRecord, - title: ncName.title, - column_name: ncName.column_name, - parentId: srcTableId, - childId: childTableId, - type: 'mm', - // aTblLinkColumns[i].typeOptions.relationship === 'many' - // ? 'mm' - // : 'hm' - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === ncName.title - )?.id; - await sMap.addToMappingTbl( - aTblLinkColumns[i].id, - ncId, - ncName.title, - ncTbl.id - ); - - // store link information in separate table - // this information will be helpful in identifying relation pair - const link = { - nc: { - title: ncName.title, - parentId: srcTableId, - childId: childTableId, - type: 'mm', - }, - aTbl: { - tblId: aTblSchema[idx].id, - ...aTblLinkColumns[i], - }, - }; - - ncLinkMappingTable.push(link); - } else { - // if link already exists, we need to change name of linked column - // to what is represented in airtable - - // 1. extract associated link information from link table - // 2. retrieve parent table information (source) - // 3. using foreign parent & child column ID, find associated mapping in child table - // 4. update column name - const x = ncLinkMappingTable.findIndex( - (x) => - x.aTbl.tblId === - aTblLinkColumns[i].typeOptions.foreignTableId && - x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId - ); - - let _perfStart = recordPerfStart(); - const childTblSchema: any = await api.dbTable.read( - ncLinkMappingTable[x].nc.childId - ); - recordPerfStats(_perfStart, 'dbTable.read'); - - _perfStart = recordPerfStart(); - const parentTblSchema: any = await api.dbTable.read( - ncLinkMappingTable[x].nc.parentId - ); - recordPerfStats(_perfStart, 'dbTable.read'); - - // fix me - // let childTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.childId] - // let parentTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.parentId] - - let parentLinkColumn = parentTblSchema.columns.find( - (col) => col.title === ncLinkMappingTable[x].nc.title - ); - - if (parentLinkColumn === undefined) { - updateMigrationSkipLog( - parentTblSchema?.title, - ncLinkMappingTable[x].nc.title, - UITypes.LinkToAnotherRecord, - 'Link error' - ); - continue; - } - - // hack // fix me - if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') { - parentLinkColumn = parentTblSchema.columns.find( - (col) => col.title === ncLinkMappingTable[x].nc.title + '_2' - ); - } - - let childLinkColumn: any = {}; - - if (parentLinkColumn.colOptions.type == 'hm') { - // for hm: - // mapping between child & parent column id is direct - // - childLinkColumn = childTblSchema.columns.find( - (col) => - col.uidt === UITypes.LinkToAnotherRecord && - col.colOptions.fk_child_column_id === - parentLinkColumn.colOptions.fk_child_column_id && - col.colOptions.fk_parent_column_id === - parentLinkColumn.colOptions.fk_parent_column_id - ); - } else { - // for mm: - // mapping between child & parent column id is inverted - // - childLinkColumn = childTblSchema.columns.find( - (col) => - col.uidt === UITypes.LinkToAnotherRecord && - col.colOptions.fk_child_column_id === - parentLinkColumn.colOptions.fk_parent_column_id && - col.colOptions.fk_parent_column_id === - parentLinkColumn.colOptions.fk_child_column_id && - col.colOptions.fk_mm_model_id === - parentLinkColumn.colOptions.fk_mm_model_id - ); - } - - // check if already a column exists with this name? - const duplicate = childTblSchema.columns.find( - (x) => x.title === aTblLinkColumns[i].name - ); - const suffix = duplicate ? '_2' : ''; - if (duplicate) - if (enableErrorLogs) - console.log(`## Duplicate ${aTblLinkColumns[i].name}`); - - // rename - // note that: current rename API requires us to send all parameters, - // not just title being renamed - const ncName = nc_getSanitizedColumnName( - childTblSchema, - aTblLinkColumns[i].name - ); - - logDetailed( - `NC API: dbTableColumn.update rename symmetric column ${ncName.title}` - ); - _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.update( - childLinkColumn.id, - { - ...childLinkColumn, - title: ncName.title, - column_name: ncName.column_name, - } - ); - recordPerfStats(_perfStart, 'dbTableColumn.update'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === aTblLinkColumns[i].name + suffix - )?.id; - await sMap.addToMappingTbl( - aTblLinkColumns[i].id, - ncId, - aTblLinkColumns[i].name + suffix, - ncTbl.id - ); - } - } - } - } - } - - async function nocoCreateLookups(aTblSchema) { - // LookUps - for (let idx = 0; idx < aTblSchema.length; idx++) { - const aTblColumns = aTblSchema[idx].columns.filter( - (x) => x.type === 'lookup' - ); - - // parent table ID - // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; - const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - const srcTableSchema = ncSchema.tablesById[srcTableId]; - - if (aTblColumns.length) { - // Lookup - for (let i = 0; i < aTblColumns.length; i++) { - logDetailed( - `[${idx + 1}/${aTblSchema.length}] Configuring Lookup :: [${ - i + 1 - }/${aTblColumns.length}] ${aTblSchema[idx].name}` - ); - - // something is not right, skip - if ( - aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds?.length - ) { - if (enableErrorLogs) - console.log(`## Invalid column IDs mapped; skip`); - - updateMigrationSkipLog( - srcTableSchema.title, - aTblColumns[i].name, - aTblColumns[i].type, - 'invalid column ID in dependency list' - ); - continue; - } - - const ncRelationColumnId = await sMap.getNcIdFromAtId( - aTblColumns[i].typeOptions.relationColumnId - ); - const ncLookupColumnId = await sMap.getNcIdFromAtId( - aTblColumns[i].typeOptions.foreignTableRollupColumnId - ); - - if ( - ncLookupColumnId === undefined || - ncRelationColumnId === undefined - ) { - aTblColumns[i]['srcTableId'] = srcTableId; - nestedLookupTbl.push(aTblColumns[i]); - continue; - } - - const ncName = nc_getSanitizedColumnName( - srcTableSchema, - aTblColumns[i].name - ); - - logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); - const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === aTblColumns[i].name - )?.id; - await sMap.addToMappingTbl( - aTblColumns[i].id, - ncId, - aTblColumns[i].name, - ncTbl.id - ); - } - } - } - - let level = 2; - let nestedCnt = 0; - while (nestedLookupTbl.length) { - // if nothing has changed from previous iteration, skip rest - if (nestedCnt === nestedLookupTbl.length) { - for (let i = 0; i < nestedLookupTbl.length; i++) { - const fTblField = - nestedLookupTbl[i].typeOptions.foreignTableRollupColumnId; - const name = aTbl_getColumnName(fTblField); - updateMigrationSkipLog( - ncSchema.tablesById[nestedLookupTbl[i].srcTableId]?.title, - nestedLookupTbl[i].name, - nestedLookupTbl[i].type, - `foreign table field not found [${name.tn}/${name.cn}]` - ); - } - if (enableErrorLogs) - console.log( - `## Failed to configure ${nestedLookupTbl.length} lookups` - ); - break; - } - - // Nested lookup - nestedCnt = nestedLookupTbl.length; - for (let i = 0; i < nestedLookupTbl.length; i++) { - const srcTableId = nestedLookupTbl[0].srcTableId; - const srcTableSchema = ncSchema.tablesById[srcTableId]; - - const ncRelationColumnId = await sMap.getNcIdFromAtId( - nestedLookupTbl[0].typeOptions.relationColumnId - ); - const ncLookupColumnId = await sMap.getNcIdFromAtId( - nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId - ); - - if ( - ncLookupColumnId === undefined || - ncRelationColumnId === undefined - ) { - continue; - } - - const ncName = nc_getSanitizedColumnName( - srcTableSchema, - nestedLookupTbl[0].name - ); - - logDetailed( - `Configuring Nested Lookup: Level-${level} [${i + 1}/${nestedCnt} ${ - ncName.title - }]` - ); - - logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); - const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === nestedLookupTbl[0].name - )?.id; - await sMap.addToMappingTbl( - nestedLookupTbl[0].id, - ncId, - nestedLookupTbl[0].name, - ncTbl.id - ); - - // remove entry - nestedLookupTbl.splice(0, 1); - } - level++; - } - } - - function getRollupNcFunction(aTblFunction) { - const fn = aTblFunction.split('(')[0]; - const aTbl_ncRollUp = { - AND: '', - ARRAYCOMPACT: '', - ARRAYJOIN: '', - ARRAYUNIQUE: '', - AVERAGE: 'average', - CONCATENATE: '', - COUNT: 'count', - COUNTA: '', - COUNTALL: '', - MAX: 'max', - MIN: 'min', - OR: '', - SUM: 'sum', - XOR: '', - }; - return aTbl_ncRollUp[fn]; - } - - //@ts-ignore - async function nocoCreateRollup(aTblSchema) { - // Rollup - for (let idx = 0; idx < aTblSchema.length; idx++) { - const aTblColumns = aTblSchema[idx].columns.filter( - (x) => x.type === 'rollup' - ); - - // parent table ID - // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; - const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - const srcTableSchema = ncSchema.tablesById[srcTableId]; - - if (aTblColumns.length) { - // rollup exist - for (let i = 0; i < aTblColumns.length; i++) { - logDetailed( - `[${idx + 1}/${aTblSchema.length}] Configuring Rollup :: [${ - i + 1 - }/${aTblColumns.length}] ${aTblSchema[idx].name}` - ); - - // fetch associated rollup function - // skip column creation if supported rollup function does not exist - const ncRollupFn = getRollupNcFunction( - aTblColumns[i].typeOptions.formulaTextParsed - ); - // const ncRollupFn = ''; - - if (ncRollupFn === '' || ncRollupFn === undefined) { - updateMigrationSkipLog( - srcTableSchema.title, - aTblColumns[i].name, - aTblColumns[i].type, - `rollup function ${aTblColumns[i].typeOptions.formulaTextParsed} not supported` - ); - continue; - } - - // something is not right, skip - if ( - aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds?.length - ) { - if (enableErrorLogs) - console.log(`## Invalid column IDs mapped; skip`); - - updateMigrationSkipLog( - srcTableSchema.title, - aTblColumns[i].name, - aTblColumns[i].type, - 'invalid column ID in dependency list' - ); - continue; - } - - const ncRelationColumnId = await sMap.getNcIdFromAtId( - aTblColumns[i].typeOptions.relationColumnId - ); - const ncRollupColumnId = await sMap.getNcIdFromAtId( - aTblColumns[i].typeOptions.foreignTableRollupColumnId - ); - - if (ncRollupColumnId === undefined) { - aTblColumns[i]['srcTableId'] = srcTableId; - nestedRollupTbl.push(aTblColumns[i]); - continue; - } - - // skip, if rollup column was pointing to another virtual column - const ncColSchema = await nc_getColumnSchema( - aTblColumns[i].typeOptions.foreignTableRollupColumnId - ); - if ( - ncColSchema?.uidt === UITypes.Formula || - ncColSchema?.uidt === UITypes.Lookup || - ncColSchema?.uidt === UITypes.Rollup || - ncColSchema?.uidt === UITypes.Checkbox - ) { - updateMigrationSkipLog( - srcTableSchema.title, - aTblColumns[i].name, - aTblColumns[i].type, - 'rollup referring to a column type not supported currently' - ); - continue; - } - - const ncName = nc_getSanitizedColumnName( - srcTableSchema, - aTblColumns[i].name - ); - - logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); - const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Rollup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_rollup_column_id: ncRollupColumnId, - rollup_function: ncRollupFn, - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === aTblColumns[i].name - )?.id; - await sMap.addToMappingTbl( - aTblColumns[i].id, - ncId, - aTblColumns[i].name, - ncTbl.id - ); - } - } - } - logDetailed(`Nested rollup: ${nestedRollupTbl.length}`); - } - - //@ts-ignore - async function nocoLookupForRollup() { - const nestedCnt = nestedLookupTbl.length; - for (let i = 0; i < nestedLookupTbl.length; i++) { - const srcTableId = nestedLookupTbl[0].srcTableId; - const srcTableSchema = ncSchema.tablesById[srcTableId]; - - const ncRelationColumnId = await sMap.getNcIdFromAtId( - nestedLookupTbl[0].typeOptions.relationColumnId - ); - const ncLookupColumnId = await sMap.getNcIdFromAtId( - nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId - ); - - if (ncLookupColumnId === undefined || ncRelationColumnId === undefined) { - continue; - } - - const ncName = nc_getSanitizedColumnName( - srcTableSchema, - nestedLookupTbl[0].name - ); - - logDetailed( - `Configuring Lookup over Rollup :: [${i + 1}/${nestedCnt}] ${ - ncName.title - }` - ); - - logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); - const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === nestedLookupTbl[0].name - )?.id; - await sMap.addToMappingTbl( - nestedLookupTbl[0].id, - ncId, - nestedLookupTbl[0].name, - ncTbl.id - ); - - // remove entry - nestedLookupTbl.splice(0, 1); - } - } - - async function nocoSetPrimary(aTblSchema) { - for (let idx = 0; idx < aTblSchema.length; idx++) { - logDetailed( - `[${idx + 1}/${aTblSchema.length}] Configuring Display value : ${ - aTblSchema[idx].name - }` - ); - - const pColId = aTblSchema[idx].primaryColumnId; - const ncColId = await sMap.getNcIdFromAtId(pColId); - - // skip primary column configuration if we field not migrated - if (ncColId) { - logDetailed(`NC API: dbTableColumn.primaryColumnSet`); - const _perfStart = recordPerfStart(); - await api.dbTableColumn.primaryColumnSet(ncColId); - recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet'); - - // update schema - const ncTblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - await updateNcTblSchemaById(ncTblId); - } - } - } - - // retrieve nc-view column ID from corresponding nc-column ID - async function nc_getViewColumnId(viewId, viewType, ncColumnId) { - // retrieve view Info - let viewDetails; - - const _perfStart = recordPerfStart(); - if (viewType === 'form') { - viewDetails = (await api.dbView.formRead(viewId)).columns; - recordPerfStats(_perfStart, 'dbView.formRead'); - } else if (viewType === 'gallery') { - viewDetails = (await api.dbView.galleryRead(viewId)).columns; - recordPerfStats(_perfStart, 'dbView.galleryRead'); - } else { - viewDetails = await api.dbView.gridColumnsList(viewId); - recordPerfStats(_perfStart, 'dbView.gridColumnsList'); - } - - return viewDetails.find((x) => x.fk_column_id === ncColumnId)?.id; - } - - ////////// Data processing - - async function nocoBaseDataProcessing_v2(sDB, table, record) { - const recordHash = hash(record); - const rec = { ...record.fields }; - - // kludge - - // trim spaces on either side of column name - // leads to error in NocoDB - Object.keys(rec).forEach((key) => { - const replacedKey = key.trim().replace(/\./g, '_'); - if (key !== replacedKey) { - rec[replacedKey] = rec[key]; - delete rec[key]; - } - }); - - // post-processing on the record - for (const [key, value] of Object.entries(rec as { [key: string]: any })) { - // retrieve datatype - const dt = table.columns.find((x) => x.title === key)?.uidt; - - // always process LTAR, Lookup, and Rollup columns as we delete the key after processing - if ( - !value && - dt !== UITypes.LinkToAnotherRecord && - dt !== UITypes.Lookup && - dt !== UITypes.Rollup - ) { - rec[key] = null; - continue; - } - - switch (dt) { - // https://www.npmjs.com/package/validator - // default value: digits_after_decimal: [2] - // if currency, set decimal place to 2 - // - case UITypes.Currency: - rec[key] = (+value).toFixed(2); - break; - - // we will pick up LTAR once all table data's are in place - case UITypes.LinkToAnotherRecord: - if (storeLinks) { - if (ncLinkDataStore[table.title][record.id] === undefined) - ncLinkDataStore[table.title][record.id] = { - id: record.id, - fields: {}, - }; - ncLinkDataStore[table.title][record.id]['fields'][key] = value; - } - delete rec[key]; - break; - - // these will be automatically populated depending on schema configuration - case UITypes.Lookup: - case UITypes.Rollup: - delete rec[key]; - break; - - case UITypes.Collaborator: - // in case of multi-collaborator, this will be an array - if (Array.isArray(value)) { - let collaborators = ''; - for (let i = 0; i < value.length; i++) { - collaborators += `${value[i]?.name} <${value[i]?.email}>, `; - rec[key] = collaborators; - } - } else rec[key] = `${value?.name} <${value?.email}>`; - break; - - case UITypes.Button: - rec[key] = `${value?.label} <${value?.url}>`; - break; - - case UITypes.DateTime: - case UITypes.CreateTime: - case UITypes.LastModifiedTime: - rec[key] = dayjs(value).format('YYYY-MM-DD HH:mm'); - break; - - case UITypes.Date: - if (/\d{5,}/.test(value)) { - // skip - rec[key] = null; - logBasic(`:: Invalid date ${value}`); - } else { - rec[key] = dayjs(value).format('YYYY-MM-DD'); - } - break; - - case UITypes.SingleSelect: - if (value === '') { - rec[key] = 'nc_empty'; - } - rec[key] = value; - break; - - case UITypes.MultiSelect: - rec[key] = value - ?.map((v) => { - if (v === '') { - return 'nc_empty'; - } - return `${v.replace(/,/g, '.')}`; - }) - .join(','); - break; - - case UITypes.Attachment: - if (!syncDB.options.syncAttachment) rec[key] = null; - else { - let tempArr = []; - - try { - logBasic( - ` :: Retrieving attachment :: ${value - ?.map((a) => a.filename?.split('?')?.[0]) - .join(', ')}` - ); - tempArr = await api.storage.uploadByUrl( - { - path: `noco/${sDB.projectName}/${table.title}/${key}`, - }, - value?.map((attachment) => ({ - fileName: attachment.filename?.split('?')?.[0], - url: attachment.url, - size: attachment.size, - mimetype: attachment.type, - })) - ); - } catch (e) { - console.log(e); - } - - rec[key] = JSON.stringify(tempArr); - } - break; - - case UITypes.SingleLineText: - // Barcode data - if (value?.text) { - rec[key] = value.text; - } - break; - - default: - break; - } - } - - // insert airtable record ID explicitly into each records - rec[ncSysFields.id] = record.id; - rec[ncSysFields.hash] = recordHash; - - return rec; - } - - // @ts-ignore - async function nocoReadDataSelected(projName, table, callback, fields) { - return new Promise((resolve, reject) => { - base(table.title) - .select({ - pageSize: 100, - // maxRecords: 100, - fields: fields, - }) - .eachPage( - async function page(records, fetchNextPage) { - // This function (`page`) will get called for each page of records. - // records.forEach(record => callback(table, record)); - logBasic( - `:: ${table.title} / ${fields} : ${ - recordCnt + 1 - } ~ ${(recordCnt += 100)}` - ); - await Promise.all( - records.map((r) => callback(projName, table, r, fields)) - ); - - // To fetch the next page of records, call `fetchNextPage`. - // If there are more records, `page` will get called again. - // If there are no more records, `done` will get called. - fetchNextPage(); - }, - function done(err) { - if (err) { - console.error(err); - reject(err); - } - resolve(null); - } - ); - }); - } - - ////////// - - function nc_isLinkExists(airtableFieldId) { - return !!ncLinkMappingTable.find( - (x) => x.aTbl.typeOptions.symmetricColumnId === airtableFieldId - ); - } - - async function nocoCreateProject(projName) { - // create empty project (XC-DB) - logDetailed(`Create Project: ${projName}`); - const _perfStart = recordPerfStart(); - ncCreatedProjectSchema = await api.project.create({ - title: projName, - }); - recordPerfStats(_perfStart, 'project.create'); - } - - async function nocoGetProject(projId) { - // create empty project (XC-DB) - logDetailed(`Getting project meta: ${projId}`); - const _perfStart = recordPerfStart(); - ncCreatedProjectSchema = await api.project.read(projId); - recordPerfStats(_perfStart, 'project.read'); - } - - async function nocoConfigureGalleryView(sDB, aTblSchema) { - if (!sDB.options.syncViews) return; - for (let idx = 0; idx < aTblSchema.length; idx++) { - const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id; - const galleryViews = aTblSchema[idx].views.filter( - (x) => x.type === 'gallery' - ); - - const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; - rtc.view.gallery += galleryViews.length; - - for (let i = 0; i < galleryViews.length; i++) { - logDetailed(` Axios fetch view-data`); - - // create view - await getViewData(galleryViews[i].id); - const viewName = aTblSchema[idx].views.find( - (x) => x.id === galleryViews[i].id - )?.name; - - logBasic( - `:: [${configuredViews + i + 1}/${rtc.view.total}] Gallery : ${ - aTblSchema[idx].name - } / ${viewName}` - ); - - logDetailed(`NC API dbView.galleryCreate :: ${viewName}`); - const _perfStart = recordPerfStart(); - await api.dbView.galleryCreate(tblId, { title: viewName }); - recordPerfStats(_perfStart, 'dbView.galleryCreate'); - - await updateNcTblSchemaById(tblId); - // syncLog(`[${idx+1}/${aTblSchema.length}][Gallery View][${i+1}/${galleryViews.length}] Create ${viewName}`) - - // await nc_configureFields(g.id, vData, aTblSchema[idx].name, viewName, 'gallery'); - } - } - } - - async function nocoConfigureFormView(sDB, aTblSchema) { - if (!sDB.options.syncViews) return; - for (let idx = 0; idx < aTblSchema.length; idx++) { - const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - const formViews = aTblSchema[idx].views.filter((x) => x.type === 'form'); - - const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; - rtc.view.form += formViews.length; - for (let i = 0; i < formViews.length; i++) { - logDetailed(` Axios fetch view-data`); - - // create view - const vData = await getViewData(formViews[i].id); - const viewName = aTblSchema[idx].views.find( - (x) => x.id === formViews[i].id - )?.name; - - logBasic( - `:: [${configuredViews + i + 1}/${rtc.view.total}] Form : ${ - aTblSchema[idx].name - } / ${viewName}` - ); - - // everything is default - let refreshMode = 'NO_REFRESH'; - let msg = 'Thank you for submitting the form!'; - let desc = ''; - - // response will not include form object if everything is default - // - if (vData.metadata?.form) { - if (vData.metadata.form?.refreshAfterSubmit) - refreshMode = vData.metadata.form.refreshAfterSubmit; - if (vData.metadata.form?.afterSubmitMessage) - msg = vData.metadata.form.afterSubmitMessage; - if (vData.metadata.form?.description) - desc = vData.metadata.form.description; - } - - const formData = { - title: viewName, - heading: viewName, - subheading: desc, - success_msg: msg, - submit_another_form: refreshMode.includes('REFRESH_BUTTON'), - show_blank_form: refreshMode.includes('AUTO_REFRESH'), - }; - - logDetailed(`NC API dbView.formCreate :: ${viewName}`); - const _perfStart = recordPerfStart(); - const f = await api.dbView.formCreate(tblId, formData); - recordPerfStats(_perfStart, 'dbView.formCreate'); - - logDetailed( - `[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${ - formViews.length - }] Create ${viewName}` - ); - - await updateNcTblSchemaById(tblId); - - logDetailed(` Configure show/hide columns`); - await nc_configureFields( - f.id, - vData, - aTblSchema[idx].name, - viewName, - 'form' - ); - } - } - } - - async function nocoConfigureGridView(sDB, aTblSchema) { - for (let idx = 0; idx < aTblSchema.length; idx++) { - const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - const gridViews = aTblSchema[idx].views.filter((x) => x.type === 'grid'); - - let viewCnt = idx; - if (syncDB.options.syncViews) - viewCnt = rtc.view.grid + rtc.view.gallery + rtc.view.form; - rtc.view.grid += gridViews.length; - - for (let i = 0; i < (sDB.options.syncViews ? gridViews.length : 1); i++) { - logDetailed(` Axios fetch view-data`); - // fetch viewData JSON - const vData = await getViewData(gridViews[i].id); - - // retrieve view name & associated NC-ID - const viewName = aTblSchema[idx].views.find( - (x) => x.id === gridViews[i].id - )?.name; - const _perfStart = recordPerfStart(); - const viewList: any = await api.dbView.list(tblId); - recordPerfStats(_perfStart, 'dbView.list'); - - let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id; - - logBasic( - `:: [${viewCnt + i + 1}/${rtc.view.total}] Grid : ${ - aTblSchema[idx].name - } / ${viewName}` - ); - - // create view (default already created) - if (i > 0) { - logDetailed(`NC API dbView.gridCreate :: ${viewName}`); - const _perfStart = recordPerfStart(); - const viewCreated = await api.dbView.gridCreate(tblId, { - title: viewName, - }); - recordPerfStats(_perfStart, 'dbView.gridCreate'); - - await updateNcTblSchemaById(tblId); - await sMap.addToMappingTbl( - gridViews[i].id, - viewCreated.id, - viewName, - tblId - ); - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Create ${viewName}`) - ncViewId = viewCreated.id; - } - - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Hide columns ${viewName}`) - logDetailed(` Configure show/hide columns`); - await nc_configureFields( - ncViewId, - vData, - aTblSchema[idx].name, - viewName, - 'grid' - ); - - // configure filters - if (vData?.filters) { - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure filters ${viewName}`) - logDetailed(` Configure filter set`); - - // skip filters if nested - if (!vData.filters.filterSet.find((x) => x?.type === 'nested')) { - await nc_configureFilters(ncViewId, vData.filters); - } - } - - // configure sort - if (vData?.lastSortsApplied?.sortSet.length) { - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure sort ${viewName}`) - logDetailed(` Configure sort set`); - await nc_configureSort(ncViewId, vData.lastSortsApplied); - } - } - } - } - - async function nocoAddUsers(aTblSchema) { - const userRoles = { - owner: 'owner', - create: 'creator', - edit: 'editor', - comment: 'commenter', - read: 'viewer', - none: 'viewer', - }; - const userList = aTblSchema.appBlanket.userInfoById; - const totalUsers = Object.keys(userList).length; - let cnt = 0; - const insertJobs: Promise[] = []; - - for (const [, value] of Object.entries( - userList as { [key: string]: any } - )) { - logDetailed( - `[${++cnt}/${totalUsers}] NC API auth.projectUserAdd :: ${value.email}` - ); - const _perfStart = recordPerfStart(); - insertJobs.push( - api.auth - .projectUserAdd(ncCreatedProjectSchema.id, { - email: value.email, - roles: userRoles[value.permissionLevel], - }) - .catch((e) => - e.response?.data?.msg - ? logBasic(`NOTICE: ${e.response.data.msg}`) - : console.log(e) - ) - ); - recordPerfStats(_perfStart, 'auth.projectUserAdd'); - } - await Promise.all(insertJobs); - } - - function updateNcTblSchema(tblSchema) { - const tblId = tblSchema.id; - - // replace entry from array if already exists - const idx = ncSchema.tables.findIndex((x) => x.id === tblId); - if (idx !== -1) ncSchema.tables.splice(idx, 1); - ncSchema.tables.push(tblSchema); - - // overwrite object if it exists - ncSchema.tablesById[tblId] = tblSchema; - } - - async function updateNcTblSchemaById(tblId) { - const _perfStart = recordPerfStart(); - const ncTbl = await api.dbTable.read(tblId); - recordPerfStats(_perfStart, 'dbTable.read'); - - updateNcTblSchema(ncTbl); - } - - /////////////////////// - - // statistics - // - const migrationStats = []; - - async function generateMigrationStats(aTblSchema) { - const migrationStatsObj = { - table_name: '', - aTbl: { - columns: 0, - links: 0, - lookup: 0, - rollup: 0, - }, - nc: { - columns: 0, - links: 0, - lookup: 0, - rollup: 0, - invalidColumn: 0, - }, - }; - for (let idx = 0; idx < aTblSchema.length; idx++) { - migrationStatsObj.table_name = aTblSchema[idx].name; - - const aTblLinkColumns = aTblSchema[idx].columns.filter( - (x) => x.type === 'foreignKey' - ); - const aTblLookup = aTblSchema[idx].columns.filter( - (x) => x.type === 'lookup' - ); - const aTblRollup = aTblSchema[idx].columns.filter( - (x) => x.type === 'rollup' - ); - - let invalidColumnId = 0; - for (let i = 0; i < aTblLookup.length; i++) { - if ( - aTblLookup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length - ) { - invalidColumnId++; - } - } - for (let i = 0; i < aTblRollup.length; i++) { - if ( - aTblRollup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length - ) { - invalidColumnId++; - } - } - - migrationStatsObj.aTbl.columns = aTblSchema[idx].columns.length; - migrationStatsObj.aTbl.links = aTblLinkColumns.length; - migrationStatsObj.aTbl.lookup = aTblLookup.length; - migrationStatsObj.aTbl.rollup = aTblRollup.length; - - const ncTbl = await nc_getTableSchema(aTblSchema[idx].name); - const linkColumn = ncTbl.columns.filter( - (x) => x.uidt === UITypes.LinkToAnotherRecord - ); - const lookup = ncTbl.columns.filter((x) => x.uidt === UITypes.Lookup); - const rollup = ncTbl.columns.filter((x) => x.uidt === UITypes.Rollup); - - // all links hardwired as m2m. m2m generates additional tables per link - // hence link/2 - migrationStatsObj.nc.columns = - ncTbl.columns.length - linkColumn.length / 2; - migrationStatsObj.nc.links = linkColumn.length / 2; - migrationStatsObj.nc.lookup = lookup.length; - migrationStatsObj.nc.rollup = rollup.length; - migrationStatsObj.nc.invalidColumn = invalidColumnId; - - const temp = JSON.parse(JSON.stringify(migrationStatsObj)); - migrationStats.push(temp); - } - - const columnSum = migrationStats.reduce((accumulator, object) => { - return accumulator + object.nc.columns; - }, 0); - const linkSum = migrationStats.reduce((accumulator, object) => { - return accumulator + object.nc.links; - }, 0); - const lookupSum = migrationStats.reduce((accumulator, object) => { - return accumulator + object.nc.lookup; - }, 0); - const rollupSum = migrationStats.reduce((accumulator, object) => { - return accumulator + object.nc.rollup; - }, 0); - - logBasic(`Quick Summary:`); - logBasic(`:: Total Tables: ${aTblSchema.length}`); - logBasic(`:: Total Columns: ${columnSum}`); - logBasic(`:: Links: ${linkSum}`); - logBasic(`:: Lookup: ${lookupSum}`); - logBasic(`:: Rollup: ${rollupSum}`); - logBasic(`:: Total Filters: ${rtc.filter}`); - logBasic(`:: Total Sort: ${rtc.sort}`); - logBasic(`:: Total Views: ${rtc.view.total}`); - logBasic(`:: Grid: ${rtc.view.grid}`); - logBasic(`:: Gallery: ${rtc.view.gallery}`); - logBasic(`:: Form: ${rtc.view.form}`); - logBasic(`:: Total Records: ${rtc.data.records}`); - logBasic(`:: Total Nested Links: ${rtc.data.nestedLinks}`); - - const duration = Date.now() - start; - logBasic(`:: Migration time: ${duration}`); - logBasic(`:: Axios fetch count: ${rtc.fetchAt.count}`); - logBasic(`:: Axios fetch time: ${rtc.fetchAt.time}`); - - if (debugMode) { - await writeJsonFileAsync('stats.json', perfStats, { spaces: 2 }); - const perflog = []; - for (let i = 0; i < perfStats.length; i++) { - perflog.push(`${perfStats[i].e}, ${perfStats[i].d}`); - } - await writeJsonFileAsync('stats.csv', perflog, { spaces: 2 }); - await writeJsonFileAsync('skip.txt', rtc.migrationSkipLog.log, { - spaces: 2, - }); - } - - T.event({ - event: 'a:airtable-import:success', - data: { - stats: { - migrationTime: duration, - totalTables: aTblSchema.length, - totalColumns: columnSum, - links: linkSum, - lookup: lookupSum, - rollup: rollupSum, - totalFilters: rtc.filter, - totalSort: rtc.sort, - view: { - total: rtc.view.total, - grid: rtc.view.grid, - gallery: rtc.view.gallery, - form: rtc.view.form, - }, - axios: { - count: rtc.fetchAt.count, - time: rtc.fetchAt.time, - }, - totalRecords: rtc.data.records, - nestedLinks: rtc.data.nestedLinks, - }, - }, - }); - } - - ////////////////////////////// - // filters - - const filterMap = { - '=': 'eq', - '!=': 'neq', - '<': 'lt', - '<=': 'lte', - '>': 'gt', - '>=': 'gte', - isEmpty: 'empty', - isNotEmpty: 'notempty', - isWithin: 'isWithin', - contains: 'like', - doesNotContain: 'nlike', - isAnyOf: 'anyof', - isNoneOf: 'nanyof', - '|': 'anyof', - '&': 'allof', - }; - - async function nc_configureFilters(viewId, f) { - for (let i = 0; i < f.filterSet.length; i++) { - const filter = f.filterSet[i]; - const colSchema = await nc_getColumnSchema(filter.columnId); - - // column not available; - // one of not migrated column; - if (!colSchema) { - updateMigrationSkipLog( - await sMap.getNcNameFromAtId(viewId), - colSchema.title, - colSchema.uidt, - `filter config skipped; column not migrated` - ); - continue; - } - const columnId = colSchema.id; - const datatype = colSchema.uidt; - const ncFilters = []; - - if (datatype === UITypes.Date || datatype === UITypes.DateTime) { - let comparison_op = null; - let comparison_sub_op = null; - let value = null; - if (['isEmpty', 'isNotEmpty'].includes(filter.operator)) { - comparison_op = filter.operator === 'isEmpty' ? 'blank' : 'notblank'; - } else { - if ('numberOfDays' in filter.value) { - value = filter.value['numberOfDays']; - } else if ('exactDate' in filter.value) { - value = filter.value['exactDate']; - } - comparison_op = filterMap[filter.operator]; - comparison_sub_op = filter.value.mode; - } - const fx = { - fk_column_id: columnId, - logical_op: f.conjunction, - comparison_op, - comparison_sub_op, - value, - }; - ncFilters.push(fx); - } - - // single-select & multi-select - else if ( - datatype === UITypes.SingleSelect || - datatype === UITypes.MultiSelect - ) { - if (filter.operator === 'doesNotContain') { - filter.operator = 'isNoneOf'; - } - // if array, break it down to multiple filters - if (Array.isArray(filter.value)) { - const fx = { - fk_column_id: columnId, - logical_op: f.conjunction, - comparison_op: filterMap[filter.operator], - value: ( - await Promise.all( - filter.value.map(async (f) => await sMap.getNcNameFromAtId(f)) - ) - ).join(','), - }; - ncFilters.push(fx); - } - // not array - add as is - else if (filter.value) { - const fx = { - fk_column_id: columnId, - logical_op: f.conjunction, - comparison_op: filterMap[filter.operator], - value: await sMap.getNcNameFromAtId(filter.value), - }; - ncFilters.push(fx); - } - } - - // other data types (number/ text/ long text/ ..) - else if (filter.value) { - const fx = { - fk_column_id: columnId, - logical_op: f.conjunction, - comparison_op: filterMap[filter.operator], - value: filter.value, - }; - ncFilters.push(fx); - } - - // insert filters - for (let i = 0; i < ncFilters.length; i++) { - const _perfStart = recordPerfStart(); - await api.dbTableFilter.create(viewId, { - ...ncFilters[i], - }); - recordPerfStats(_perfStart, 'dbTableFilter.create'); - - rtc.filter++; - } - } - } - - async function nc_configureSort(viewId, s) { - for (let i = 0; i < s.sortSet.length; i++) { - const columnId = (await nc_getColumnSchema(s.sortSet[i].columnId))?.id; - - if (columnId) { - const _perfStart = recordPerfStart(); - await api.dbTableSort.create(viewId, { - fk_column_id: columnId, - direction: s.sortSet[i].ascending ? 'asc' : 'dsc', - }); - recordPerfStats(_perfStart, 'dbTableSort.create'); - } - rtc.sort++; - } - } - - async function nc_configureFields(_viewId, _c, tblName, viewName, viewType?) { - // force hide PK column - const hiddenColumns = [ncSysFields.id, ncSysFields.hash]; - const c = _c.columnOrder; - - // column order corrections - // retrieve table schema - const ncTbl = await nc_getTableSchema(tblName); - // retrieve view ID - const viewId = ncTbl.views.find((x) => x.title === viewName).id; - let viewDetails; - - const _perfStart = recordPerfStart(); - if (viewType === 'form') { - viewDetails = (await api.dbView.formRead(viewId)).columns; - recordPerfStats(_perfStart, 'dbView.formRead'); - } else if (viewType === 'gallery') { - viewDetails = (await api.dbView.galleryRead(viewId)).columns; - recordPerfStats(_perfStart, 'dbView.galleryRead'); - } else { - viewDetails = await api.dbView.gridColumnsList(viewId); - recordPerfStats(_perfStart, 'dbView.gridColumnsList'); - } - - // nc-specific columns; default hide. - for (let j = 0; j < hiddenColumns.length; j++) { - const ncColumnId = ncTbl.columns.find( - (x) => x.title === hiddenColumns[j] - ).id; - const ncViewColumnId = viewDetails.find( - (x) => x.fk_column_id === ncColumnId - )?.id; - // const ncViewColumnId = await nc_getViewColumnId( - // viewId, - // viewType, - // ncColumnId - // ); - if (ncViewColumnId === undefined) continue; - - // first two positions held by record id & record hash - const _perfStart = recordPerfStart(); - await api.dbViewColumn.update(viewId, ncViewColumnId, { - show: false, - order: j + 1 + c.length, - }); - recordPerfStats(_perfStart, 'dbViewColumn.update'); - } - - // rest of the columns from airtable- retain order & visibility property - for (let j = 0; j < c.length; j++) { - const ncColumnId = await sMap.getNcIdFromAtId(c[j].columnId); - const ncViewColumnId = await nc_getViewColumnId( - viewId, - viewType, - ncColumnId - ); - if (ncViewColumnId === undefined) continue; - - // first two positions held by record id & record hash - const configData = { show: c[j].visibility, order: j + 1 }; - if (viewType === 'form') { - if (_c?.metadata?.form?.fieldsByColumnId?.[c[j].columnId]) { - const x = _c.metadata.form.fieldsByColumnId[c[j].columnId]; - const formData = { ...configData }; - if (x?.title) formData[`label`] = x.title; - if (x?.required) formData[`required`] = x.required; - if (x?.description) formData[`description`] = x.description; - const _perfStart = recordPerfStart(); - await api.dbView.formColumnUpdate(ncViewColumnId, formData); - recordPerfStats(_perfStart, 'dbView.formColumnUpdate'); - } - } - const _perfStart = recordPerfStart(); - await api.dbViewColumn.update(viewId, ncViewColumnId, configData); - recordPerfStats(_perfStart, 'dbViewColumn.update'); - } - } - - /////////////////////////////////////////////////////////////////////////////// - let recordCnt = 0; - try { - logBasic('SDK initialized'); - api = new Api({ - baseURL: syncDB.baseURL, - headers: { - 'xc-auth': syncDB.authToken, - }, - }); - - logDetailed('Project initialization started'); - // delete project if already exists - if (debugMode) await init(syncDB); - - logDetailed('Project initialized'); - - logBasic('Retrieving Airtable schema'); - // read schema file - const schema = await getAirtableSchema(syncDB); - const aTblSchema = schema.tableSchemas; - logDetailed('Project schema extraction completed'); - - if (!syncDB.projectId) { - if (!syncDB.projectName) - throw new Error('Project name or id not provided'); - // create empty project - await nocoCreateProject(syncDB.projectName); - logDetailed('Project created'); - } else { - await nocoGetProject(syncDB.projectId); - syncDB.projectName = ncCreatedProjectSchema?.title; - syncDB.baseId = syncDB.baseId || ncCreatedProjectSchema.bases[0].id; - logDetailed('Getting existing project meta'); - } - - logBasic('Importing Tables...'); - // prepare table schema (base) - await nocoCreateBaseSchema(aTblSchema); - logDetailed('Table creation completed'); - - logDetailed('Configuring Links'); - // add LTAR - await nocoCreateLinkToAnotherRecord(aTblSchema); - logDetailed('Migrating LTAR columns completed'); - - if (syncDB.options.syncLookup) { - logDetailed(`Configuring Lookup`); - // add look-ups - await nocoCreateLookups(aTblSchema); - logDetailed('Migrating Lookup columns completed'); - } - - if (syncDB.options.syncRollup) { - logDetailed('Configuring Rollup'); - // add roll-ups - await nocoCreateRollup(aTblSchema); - logDetailed('Migrating Rollup columns completed'); - - if (syncDB.options.syncLookup) { - logDetailed('Migrating Lookup form Rollup columns'); - // lookups for rollup - await nocoLookupForRollup(); - logDetailed('Migrating Lookup form Rollup columns completed'); - } - } - logDetailed('Configuring Display Value column'); - // configure Display Value - await nocoSetPrimary(aTblSchema); - logDetailed('Configuring Display Value column completed'); - - logBasic('Configuring User(s)'); - // add users - await nocoAddUsers(schema); - logDetailed('Adding users completed'); - - // hide-fields - // await nocoReconfigureFields(aTblSchema); - - logBasic('Syncing views'); - // configure views - await nocoConfigureGridView(syncDB, aTblSchema); - await nocoConfigureFormView(syncDB, aTblSchema); - await nocoConfigureGalleryView(syncDB, aTblSchema); - logDetailed('Syncing views completed'); - - if (syncDB.options.syncData) { - try { - // await nc_DumpTableSchema(); - const _perfStart = recordPerfStart(); - const ncTblList = await api.base.tableList( - ncCreatedProjectSchema.id, - syncDB.baseId - ); - recordPerfStats(_perfStart, 'base.tableList'); - - logBasic('Reading Records...'); - - const recordsMap = {}; - - for (let i = 0; i < ncTblList.list.length; i++) { - // not a migrated table, skip - if ( - undefined === - aTblSchema.find((x) => x.name === ncTblList.list[i].title) - ) - continue; - - const _perfStart = recordPerfStart(); - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); - recordPerfStats(_perfStart, 'dbTable.read'); - - recordCnt = 0; - // await nocoReadData(syncDB, ncTbl); - - recordsMap[ncTbl.id] = await importData({ - projectName: syncDB.projectName, - table: ncTbl, - base, - api, - logBasic, - nocoBaseDataProcessing_v2, - sDB: syncDB, - logDetailed, - }); - rtc.data.records += await recordsMap[ncTbl.id].getCount(); - - logDetailed(`Data inserted from ${ncTbl.title}`); - } - - logBasic('Configuring Record Links...'); - for (let i = 0; i < ncTblList.list.length; i++) { - // not a migrated table, skip - if ( - undefined === - aTblSchema.find((x) => x.name === ncTblList.list[i].title) - ) - continue; - - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); - - rtc.data.nestedLinks += await importLTARData({ - table: ncTbl, - projectName: syncDB.projectName, - api, - base, - fields: null, //Object.values(tblLinkGroup).flat(), - logBasic, - insertedAssocRef, - logDetailed, - records: recordsMap[ncTbl.id], - atNcAliasRef, - ncLinkMappingTable, - }); - } - - if (storeLinks) { - // const insertJobs: Promise[] = []; - // for (const [pTitle, v] of Object.entries(ncLinkDataStore)) { - // logBasic(`:: ${pTitle}`); - // for (const [, record] of Object.entries(v)) { - // const tbl = ncTblList.list.find(a => a.title === pTitle); - // await nocoLinkProcessing(syncDB.projectName, tbl, record, 0); - // // insertJobs.push( - // // nocoLinkProcessing(syncDB.projectName, tbl, record, 0) - // // ); - // } - // } - // await Promise.all(insertJobs); - // await nocoLinkProcessing(syncDB.projectName, 0, 0, 0); - } else { - // // create link groups (table: link fields) - // // const tblLinkGroup = {}; - // // for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { - // // const x = ncLinkMappingTable[idx]; - // // if (tblLinkGroup[x.aTbl.tblId] === undefined) - // // tblLinkGroup[x.aTbl.tblId] = [x.aTbl.name]; - // // else tblLinkGroup[x.aTbl.tblId].push(x.aTbl.name); - // // } - // // - // // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); - // // - // // await importLTARData({ - // // table: ncTbl, - // // projectName: syncDB.projectName, - // // api, - // // base, - // // fields: Object.values(tblLinkGroup).flat(), - // // logBasic - // // }); - // for (const [k, v] of Object.entries(tblLinkGroup)) { - // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); - // - // // not a migrated table, skip - // if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) - // continue; - // - // recordCnt = 0; - // await nocoReadDataSelected( - // syncDB.projectName, - // ncTbl, - // async (projName, table, record, _field) => { - // await nocoLinkProcessing(projName, table, record, _field); - // }, - // v - // ); - // } - } - } catch (error) { - logDetailed( - `There was an error while migrating data! Please make sure your API key (${syncDB.apiKey}) is correct.` - ); - logDetailed(`Error: ${error}`); - } - } - if (generate_migrationStats) { - await generateMigrationStats(aTblSchema); - } - } catch (e) { - if (e.response?.data?.msg) { - T.event({ - event: 'a:airtable-import:error', - data: { error: e.response.data.msg }, - }); - throw new Error(e.response.data.msg); - } - throw e; - } -}; - -export function getUniqueNameGenerator(defaultName = 'name') { - const namesRef = {}; - - return (initName: string = defaultName): string => { - let name = initName === '_' ? defaultName : initName; - let c = 0; - while (name in namesRef) { - name = `${initName}_${++c}`; - } - namesRef[name] = true; - return name; - }; -} - -export interface AirtableSyncConfig { - id: string; - baseURL: string; - authToken: string; - projectName?: string; - projectId?: string; - baseId?: string; - apiKey: string; - shareId: string; - options: { - syncViews: boolean; - syncData: boolean; - syncRollup: boolean; - syncLookup: boolean; - syncFormula: boolean; - syncAttachment: boolean; - }; -} diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/controllers/sync/helpers/readAndProcessData.ts deleted file mode 100644 index ad004205d1..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/readAndProcessData.ts +++ /dev/null @@ -1,338 +0,0 @@ -import { AirtableBase } from 'airtable/lib/airtable_base'; -import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk'; -import EntityMap from './EntityMap'; - -const BULK_DATA_BATCH_SIZE = 500; -const ASSOC_BULK_DATA_BATCH_SIZE = 1000; -const BULK_PARALLEL_PROCESS = 5; - -async function readAllData({ - table, - fields, - base, - logBasic = (_str) => {}, -}: { - table: { title?: string }; - fields?; - base: AirtableBase; - logBasic?: (string) => void; - logDetailed?: (string) => void; -}): Promise { - return new Promise((resolve, reject) => { - let data = null; - - const selectParams: any = { - pageSize: 100, - }; - - if (fields) selectParams.fields = fields; - - base(table.title) - .select(selectParams) - .eachPage( - async function page(records, fetchNextPage) { - if (!data) { - data = new EntityMap(); - await data.init(); - } - - for await (const record of records) { - await data.addRow({ id: record.id, ...record.fields }); - } - - const tmpLength = await data.getCount(); - - logBasic( - `:: Reading '${table.title}' data :: ${Math.max( - 1, - tmpLength - records.length - )} - ${tmpLength}` - ); - - // To fetch the next page of records, call `fetchNextPage`. - // If there are more records, `page` will get called again. - // If there are no more records, `done` will get called. - fetchNextPage(); - }, - async function done(err) { - if (err) { - console.error(err); - return reject(err); - } - resolve(data); - } - ); - }); -} - -export async function importData({ - projectName, - table, - base, - api, - nocoBaseDataProcessing_v2, - sDB, - logDetailed = (_str) => {}, - logBasic = (_str) => {}, -}: { - projectName: string; - table: { title?: string; id?: string }; - fields?; - base: AirtableBase; - logBasic: (string) => void; - logDetailed: (string) => void; - api: Api; - nocoBaseDataProcessing_v2; - sDB; -}): Promise { - try { - // @ts-ignore - const records = await readAllData({ - table, - base, - logDetailed, - logBasic, - }); - - await new Promise(async (resolve) => { - const readable = records.getStream(); - const allRecordsCount = await records.getCount(); - const promises = []; - let tempData = []; - let importedCount = 0; - let activeProcess = 0; - readable.on('data', async (record) => { - promises.push( - new Promise(async (resolve) => { - activeProcess++; - if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause(); - const { id: rid, ...fields } = record; - const r = await nocoBaseDataProcessing_v2(sDB, table, { - id: rid, - fields, - }); - tempData.push(r); - - if (tempData.length >= BULK_DATA_BATCH_SIZE) { - let insertArray = tempData.splice(0, tempData.length); - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - table.id, - insertArray - ); - logBasic( - `:: Importing '${ - table.title - }' data :: ${importedCount} - ${Math.min( - importedCount + BULK_DATA_BATCH_SIZE, - allRecordsCount - )}` - ); - importedCount += insertArray.length; - insertArray = []; - } - activeProcess--; - if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume(); - resolve(true); - }) - ); - }); - readable.on('end', async () => { - await Promise.all(promises); - if (tempData.length > 0) { - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - table.id, - tempData - ); - logBasic( - `:: Importing '${ - table.title - }' data :: ${importedCount} - ${Math.min( - importedCount + BULK_DATA_BATCH_SIZE, - allRecordsCount - )}` - ); - importedCount += tempData.length; - tempData = []; - } - resolve(true); - }); - }); - - return records; - } catch (e) { - console.log(e); - return null; - } -} - -export async function importLTARData({ - table, - fields, - base, - api, - projectName, - insertedAssocRef = {}, - logDetailed = (_str) => {}, - logBasic = (_str) => {}, - records, - atNcAliasRef, - ncLinkMappingTable, -}: { - projectName: string; - table: { title?: string; id?: string }; - fields; - base: AirtableBase; - logDetailed: (string) => void; - logBasic: (string) => void; - api: Api; - insertedAssocRef: { [assocTableId: string]: boolean }; - records?: EntityMap; - atNcAliasRef: { - [ncTableId: string]: { - [ncTitle: string]: string; - }; - }; - ncLinkMappingTable: Record>[]; -}) { - const assocTableMetas: Array<{ - modelMeta: { id?: string; title?: string }; - colMeta: { title?: string }; - curCol: { title?: string }; - refCol: { title?: string }; - }> = []; - const allData = - records || - (await readAllData({ - table, - fields, - base, - logDetailed, - logBasic, - })); - - const modelMeta: any = await api.dbTable.read(table.id); - - for (const colMeta of modelMeta.columns) { - // skip columns which are not LTAR and Many to many - if ( - colMeta.uidt !== UITypes.LinkToAnotherRecord || - colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY - ) { - continue; - } - - // skip if already inserted - if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue; - - // self links: skip if the column under consideration is the add-on column NocoDB creates - if (ncLinkMappingTable.every((a) => a.nc.title !== colMeta.title)) continue; - - // mark as inserted - insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; - - const assocModelMeta: TableType = (await api.dbTable.read( - colMeta.colOptions.fk_mm_model_id - )) as any; - - // extract associative table and columns meta - assocTableMetas.push({ - modelMeta: assocModelMeta, - colMeta, - curCol: assocModelMeta.columns.find( - (c) => c.id === colMeta.colOptions.fk_mm_child_column_id - ), - refCol: assocModelMeta.columns.find( - (c) => c.id === colMeta.colOptions.fk_mm_parent_column_id - ), - }); - } - - let nestedLinkCnt = 0; - // Iterate over all related M2M associative table - for await (const assocMeta of assocTableMetas) { - let assocTableData = []; - let importedCount = 0; - - // extract insert data from records - await new Promise((resolve) => { - const promises = []; - const readable = allData.getStream(); - let activeProcess = 0; - readable.on('data', async (record) => { - promises.push( - new Promise(async (resolve) => { - activeProcess++; - if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause(); - const { id: _atId, ...rec } = record; - - // todo: use actual alias instead of sanitized - assocTableData.push( - ...( - rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || [] - ).map((id) => ({ - [assocMeta.curCol.title]: record.id, - [assocMeta.refCol.title]: id, - })) - ); - - if (assocTableData.length >= ASSOC_BULK_DATA_BATCH_SIZE) { - let insertArray = assocTableData.splice(0, assocTableData.length); - logBasic( - `:: Importing '${ - table.title - }' LTAR data :: ${importedCount} - ${Math.min( - importedCount + ASSOC_BULK_DATA_BATCH_SIZE, - insertArray.length - )}` - ); - - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - assocMeta.modelMeta.id, - insertArray - ); - - importedCount += insertArray.length; - insertArray = []; - } - activeProcess--; - if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume(); - resolve(true); - }) - ); - }); - readable.on('end', async () => { - await Promise.all(promises); - if (assocTableData.length >= 0) { - logBasic( - `:: Importing '${ - table.title - }' LTAR data :: ${importedCount} - ${Math.min( - importedCount + ASSOC_BULK_DATA_BATCH_SIZE, - assocTableData.length - )}` - ); - - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - assocMeta.modelMeta.id, - assocTableData - ); - - importedCount += assocTableData.length; - assocTableData = []; - } - resolve(true); - }); - }); - - nestedLinkCnt += importedCount; - } - return nestedLinkCnt; -} diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/syncMap.ts b/packages/nocodb/src/lib/controllers/sync/helpers/syncMap.ts deleted file mode 100644 index 7855bbaa2c..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/syncMap.ts +++ /dev/null @@ -1,31 +0,0 @@ -export const mapTbl = {}; - -// static mapping records between aTblId && ncId -export const addToMappingTbl = function addToMappingTbl( - aTblId, - ncId, - ncName, - parent? -) { - mapTbl[aTblId] = { - ncId: ncId, - ncParent: parent, - // name added to assist in quick debug - ncName: ncName, - }; -}; - -// get NcID from airtable ID -export const getNcIdFromAtId = function getNcIdFromAtId(aId) { - return mapTbl[aId]?.ncId; -}; - -// get nc Parent from airtable ID -export const getNcParentFromAtId = function getNcParentFromAtId(aId) { - return mapTbl[aId]?.ncParent; -}; - -// get nc-title from airtable ID -export const getNcNameFromAtId = function getNcNameFromAtId(aId) { - return mapTbl[aId]?.ncName; -}; diff --git a/packages/nocodb/src/lib/controllers/sync/importApis.ts b/packages/nocodb/src/lib/controllers/sync/importApis.ts deleted file mode 100644 index 029caa0657..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/importApis.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { Request, Router } from 'express'; -// import { Queue } from 'bullmq'; -// import axios from 'axios'; -import catchError, { NcError } from '../../meta/helpers/catchError'; -import { Server } from 'socket.io'; -import NocoJobs from '../../jobs/NocoJobs'; -import job, { AirtableSyncConfig } from './helpers/job'; -import SyncSource from '../../models/SyncSource'; -import Noco from '../../Noco'; -import { genJwt } from '../userApi/helpers'; -const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB'; -const AIRTABLE_PROGRESS_JOB = 'AIRTABLE_PROGRESS_JOB'; - -enum SyncStatus { - PROGRESS = 'PROGRESS', - COMPLETED = 'COMPLETED', - FAILED = 'FAILED', -} - -export default ( - router: Router, - sv: Server, - jobs: { [id: string]: { last_message: any } } -) => { - // add importer job handler and progress notification job handler - NocoJobs.jobsMgr.addJobWorker(AIRTABLE_IMPORT_JOB, job); - NocoJobs.jobsMgr.addJobWorker( - AIRTABLE_PROGRESS_JOB, - ({ payload, progress }) => { - sv.to(payload?.id).emit('progress', { - msg: progress?.msg, - level: progress?.level, - status: progress?.status, - }); - - if (payload?.id in jobs) { - jobs[payload?.id].last_message = { - msg: progress?.msg, - level: progress?.level, - status: progress?.status, - }; - } - } - ); - - NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => { - NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { - payload, - progress: { - msg: progress?.msg, - level: progress?.level, - status: progress?.status, - }, - }); - }); - NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, (payload) => { - NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { - payload, - progress: { - msg: 'Complete!', - status: SyncStatus.COMPLETED, - }, - }); - delete jobs[payload?.id]; - }); - NocoJobs.jobsMgr.addFailureCbk(AIRTABLE_IMPORT_JOB, (payload, error: any) => { - NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { - payload, - progress: { - msg: error?.message || 'Failed due to some internal error', - status: SyncStatus.FAILED, - }, - }); - delete jobs[payload?.id]; - }); - - router.post( - '/api/v1/db/meta/import/airtable', - catchError((req, res) => { - NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { - id: req.query.id, - ...req.body, - }); - res.json({}); - }) - ); - router.post( - '/api/v1/db/meta/syncs/:syncId/trigger', - catchError(async (req: Request, res) => { - if (req.params.syncId in jobs) { - NcError.badRequest('Sync already in progress'); - } - - const syncSource = await SyncSource.get(req.params.syncId); - - const user = await syncSource.getUser(); - const token = genJwt(user, Noco.getConfig()); - - // Treat default baseUrl as siteUrl from req object - let baseURL = (req as any).ncSiteUrl; - - // if environment value avail use it - // or if it's docker construct using `PORT` - if (process.env.NC_BASEURL_INTERNAL) { - baseURL = process.env.NC_BASEURL_INTERNAL; - } else if (process.env.NC_DOCKER) { - baseURL = `http://localhost:${process.env.PORT || 8080}`; - } - - setTimeout(() => { - NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { - id: req.params.syncId, - ...(syncSource?.details || {}), - projectId: syncSource.project_id, - baseId: syncSource.base_id, - authToken: token, - baseURL, - }); - }, 1000); - - jobs[req.params.syncId] = { - last_message: { - msg: 'Sync started', - }, - }; - res.json({}); - }) - ); - router.post( - '/api/v1/db/meta/syncs/:syncId/abort', - catchError(async (req: Request, res) => { - if (req.params.syncId in jobs) { - delete jobs[req.params.syncId]; - } - res.json({}); - }) - ); -}; diff --git a/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts b/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts deleted file mode 100644 index 9da8d70a70..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Request, Response, Router } from 'express'; - -import SyncSource from '../../../models/SyncSource'; -import { T } from 'nc-help'; -import { PagedResponseImpl } from '../../helpers/PagedResponse'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import Project from '../../../models/Project'; - -export async function syncSourceList(req: Request, res: Response) { - // todo: pagination - res.json( - new PagedResponseImpl( - await SyncSource.list(req.params.projectId, req.params.baseId) - ) - ); -} - -export async function syncCreate(req: Request, res: Response) { - T.emit('evt', { evt_type: 'webhooks:created' }); - const project = await Project.getWithInfo(req.params.projectId); - - const sync = await SyncSource.insert({ - ...req.body, - fk_user_id: (req as any).user.id, - base_id: req.params.baseId ? req.params.baseId : project.bases[0].id, - project_id: req.params.projectId, - }); - res.json(sync); -} - -export async function syncDelete(req: Request, res: Response) { - T.emit('evt', { evt_type: 'webhooks:deleted' }); - res.json(await SyncSource.delete(req.params.syncId)); -} - -export async function syncUpdate(req: Request, res: Response) { - T.emit('evt', { evt_type: 'webhooks:updated' }); - - res.json(await SyncSource.update(req.params.syncId, req.body)); -} - -const router = Router({ mergeParams: true }); - -router.get( - '/api/v1/db/meta/projects/:projectId/syncs', - ncMetaAclMw(syncSourceList, 'syncSourceList') -); -router.post( - '/api/v1/db/meta/projects/:projectId/syncs', - ncMetaAclMw(syncCreate, 'syncSourceCreate') -); -router.get( - '/api/v1/db/meta/projects/:projectId/syncs/:baseId', - ncMetaAclMw(syncSourceList, 'syncSourceList') -); -router.post( - '/api/v1/db/meta/projects/:projectId/syncs/:baseId', - ncMetaAclMw(syncCreate, 'syncSourceCreate') -); -router.delete( - '/api/v1/db/meta/syncs/:syncId', - ncMetaAclMw(syncDelete, 'syncSourceDelete') -); -router.patch( - '/api/v1/db/meta/syncs/:syncId', - ncMetaAclMw(syncUpdate, 'syncSourceUpdate') -); - -export default router; diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index cc8ace772b..d8987ef3cb 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -1,7 +1,6 @@ import { Request, Response, Router } from 'express'; import { TableListType, TableReqType, TableType } from 'nocodb-sdk'; import { getAjvValidatorMw } from '../meta/api/helpers'; -import { tableUpdate } from '../meta/api/tableApis'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; @@ -59,6 +58,17 @@ export async function tableReorder(req: Request, res: Response) { ); } + + +export async function tableUpdate(req: Request, res) { + tableService.tableUpdate({ + tableId: req.params.tableId, + table: req.body, + }) + + res.json({ msg: 'success' }) +} + const router = Router({ mergeParams: true }); router.get( '/api/v1/db/meta/projects/:projectId/tables', diff --git a/packages/nocodb/src/lib/controllers/userApi/helpers.ts b/packages/nocodb/src/lib/controllers/userApi/helpers.ts deleted file mode 100644 index 387b51170e..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/helpers.ts +++ /dev/null @@ -1,23 +0,0 @@ -import * as jwt from 'jsonwebtoken'; -import crypto from 'crypto'; -import User from '../../models/User'; -import { NcConfig } from '../../../interface/config'; - -export function genJwt(user: User, config: NcConfig) { - return jwt.sign( - { - email: user.email, - firstname: user.firstname, - lastname: user.lastname, - id: user.id, - roles: user.roles, - token_version: user.token_version, - }, - config.auth.jwt.secret, - config.auth.jwt.options - ); -} - -export function randomTokenString(): string { - return crypto.randomBytes(40).toString('hex'); -} diff --git a/packages/nocodb/src/lib/controllers/userApi/index.ts b/packages/nocodb/src/lib/controllers/userApi/index.ts deleted file mode 100644 index c4a5003430..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './userApis'; diff --git a/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts b/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts deleted file mode 100644 index 185fd155ae..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts +++ /dev/null @@ -1,283 +0,0 @@ -import User from '../../../models/User'; -import { v4 as uuidv4 } from 'uuid'; -import { promisify } from 'util'; - -import bcrypt from 'bcryptjs'; -import Noco from '../../../Noco'; -import { CacheScope, MetaTable } from '../../../utils/globals'; -import ProjectUser from '../../../models/ProjectUser'; -import { validatePassword } from 'nocodb-sdk'; -import boxen from 'boxen'; -import NocoCache from '../../../cache/NocoCache'; -import { T } from 'nc-help'; - -const { isEmail } = require('validator'); -const rolesLevel = { owner: 0, creator: 1, editor: 2, commenter: 3, viewer: 4 }; - -export default async function initAdminFromEnv(_ncMeta = Noco.ncMeta) { - if (process.env.NC_ADMIN_EMAIL && process.env.NC_ADMIN_PASSWORD) { - if (!isEmail(process.env.NC_ADMIN_EMAIL?.trim())) { - console.log( - '\n', - boxen( - `Provided admin email '${process.env.NC_ADMIN_EMAIL}' is not valid`, - { - title: 'Invalid admin email', - padding: 1, - borderStyle: 'double', - titleAlignment: 'center', - borderColor: 'red', - } - ), - '\n' - ); - process.exit(1); - } - - const { valid, error, hint } = validatePassword( - process.env.NC_ADMIN_PASSWORD - ); - if (!valid) { - console.log( - '\n', - boxen(`${error}${hint ? `\n\n${hint}` : ''}`, { - title: 'Invalid admin password', - padding: 1, - borderStyle: 'double', - titleAlignment: 'center', - borderColor: 'red', - }), - '\n' - ); - process.exit(1); - } - - let ncMeta; - try { - ncMeta = await _ncMeta.startTransaction(); - const email = process.env.NC_ADMIN_EMAIL.toLowerCase().trim(); - - const salt = await promisify(bcrypt.genSalt)(10); - const password = await promisify(bcrypt.hash)( - process.env.NC_ADMIN_PASSWORD, - salt - ); - const email_verification_token = uuidv4(); - const roles = 'user,super'; - - // if super admin not present - if (await User.isFirst(ncMeta)) { - // roles = 'owner,creator,editor' - T.emit('evt', { - evt_type: 'project:invite', - count: 1, - }); - - await User.insert( - { - firstname: '', - lastname: '', - email, - salt, - password, - email_verification_token, - roles, - }, - ncMeta - ); - } else { - const salt = await promisify(bcrypt.genSalt)(10); - const password = await promisify(bcrypt.hash)( - process.env.NC_ADMIN_PASSWORD, - salt - ); - const email_verification_token = uuidv4(); - const superUser = await ncMeta.metaGet2(null, null, MetaTable.USERS, { - roles: 'user,super', - }); - - if (!superUser?.id) { - const existingUserWithNewEmail = await User.getByEmail(email, ncMeta); - if (existingUserWithNewEmail?.id) { - // clear cache - await NocoCache.delAll( - CacheScope.USER, - `${existingUserWithNewEmail.email}___*` - ); - await NocoCache.del( - `${CacheScope.USER}:${existingUserWithNewEmail.id}` - ); - await NocoCache.del( - `${CacheScope.USER}:${existingUserWithNewEmail.email}` - ); - - // Update email and password of super admin account - await User.update( - existingUserWithNewEmail.id, - { - salt, - email, - password, - email_verification_token, - token_version: null, - refresh_token: null, - roles, - }, - ncMeta - ); - } else { - T.emit('evt', { - evt_type: 'project:invite', - count: 1, - }); - - await User.insert( - { - firstname: '', - lastname: '', - email, - salt, - password, - email_verification_token, - roles, - }, - ncMeta - ); - } - } else if (email !== superUser.email) { - // update admin email and password and migrate projects - // if user already present and associated with some project - - // check user account already present with the new admin email - const existingUserWithNewEmail = await User.getByEmail(email, ncMeta); - - if (existingUserWithNewEmail?.id) { - // get all project access belongs to the existing account - // and migrate to the admin account - const existingUserProjects = await ncMeta.metaList2( - null, - null, - MetaTable.PROJECT_USERS, - { - condition: { fk_user_id: existingUserWithNewEmail.id }, - } - ); - - for (const existingUserProject of existingUserProjects) { - const userProject = await ProjectUser.get( - existingUserProject.project_id, - superUser.id, - ncMeta - ); - - // if admin user already have access to the project - // then update role based on the highest access level - if (userProject) { - if ( - rolesLevel[userProject.roles] > - rolesLevel[existingUserProject.roles] - ) { - await ProjectUser.update( - userProject.project_id, - superUser.id, - existingUserProject.roles, - ncMeta - ); - } - } else { - // if super doesn't have access then add the access - await ProjectUser.insert( - { - ...existingUserProject, - fk_user_id: superUser.id, - }, - ncMeta - ); - } - // delete the old project access entry from DB - await ProjectUser.delete( - existingUserProject.project_id, - existingUserProject.fk_user_id, - ncMeta - ); - } - - // delete existing user - await ncMeta.metaDelete( - null, - null, - MetaTable.USERS, - existingUserWithNewEmail.id - ); - - // clear cache - await NocoCache.delAll( - CacheScope.USER, - `${existingUserWithNewEmail.email}___*` - ); - await NocoCache.del( - `${CacheScope.USER}:${existingUserWithNewEmail.id}` - ); - await NocoCache.del( - `${CacheScope.USER}:${existingUserWithNewEmail.email}` - ); - - // Update email and password of super admin account - await User.update( - superUser.id, - { - salt, - email, - password, - email_verification_token, - token_version: null, - refresh_token: null, - }, - ncMeta - ); - } else { - // if email's are not different update the password and hash - await User.update( - superUser.id, - { - salt, - email, - password, - email_verification_token, - token_version: null, - refresh_token: null, - }, - ncMeta - ); - } - } else { - const newPasswordHash = await promisify(bcrypt.hash)( - process.env.NC_ADMIN_PASSWORD, - superUser.salt - ); - - if (newPasswordHash !== superUser.password) { - // if email's are same and passwords are different - // then update the password and token version - await User.update( - superUser.id, - { - salt, - password, - email_verification_token, - token_version: null, - refresh_token: null, - }, - ncMeta - ); - } - } - } - await ncMeta.commit(); - } catch (e) { - console.log('Error occurred while updating/creating admin user'); - console.log(e); - await ncMeta.rollback(e); - } - } -} diff --git a/packages/nocodb/src/lib/controllers/userApi/initStrategies.ts b/packages/nocodb/src/lib/controllers/userApi/initStrategies.ts deleted file mode 100644 index 85fc8bbf9c..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/initStrategies.ts +++ /dev/null @@ -1,336 +0,0 @@ -import { OrgUserRoles } from 'nocodb-sdk'; -import User from '../../models/User'; -import ProjectUser from '../../models/ProjectUser'; -import { promisify } from 'util'; -import { Strategy as CustomStrategy } from 'passport-custom'; -import passport from 'passport'; -import passportJWT from 'passport-jwt'; -import { Strategy as AuthTokenStrategy } from 'passport-auth-token'; -import { Strategy as GoogleStrategy } from 'passport-google-oauth20'; - -const PassportLocalStrategy = require('passport-local').Strategy; -const ExtractJwt = passportJWT.ExtractJwt; -const JwtStrategy = passportJWT.Strategy; - -const jwtOptions = { - jwtFromRequest: ExtractJwt.fromHeader('xc-auth'), -}; - -import bcrypt from 'bcryptjs'; -import Project from '../../models/Project'; -import NocoCache from '../../cache/NocoCache'; -import { CacheGetType, CacheScope } from '../../utils/globals'; -import ApiToken from '../../models/ApiToken'; -import Noco from '../../Noco'; -import Plugin from '../../models/Plugin'; -import { registerNewUserIfAllowed } from './userApis'; - -export function initStrategies(router): void { - passport.use( - 'authtoken', - new AuthTokenStrategy( - { headerFields: ['xc-token'], passReqToCallback: true }, - (req, token, done) => { - ApiToken.getByToken(token) - .then((apiToken) => { - if (!apiToken) { - return done({ msg: 'Invalid token' }); - } - - if (!apiToken.fk_user_id) return done(null, { roles: 'editor' }); - User.get(apiToken.fk_user_id) - .then((user) => { - user['is_api_token'] = true; - if (req.ncProjectId) { - ProjectUser.get(req.ncProjectId, user.id) - .then(async (projectUser) => { - user.roles = projectUser?.roles || user.roles; - user.roles = - user.roles === 'owner' ? 'owner,creator' : user.roles; - // + (user.roles ? `,${user.roles}` : ''); - // todo : cache - // await NocoCache.set(`${CacheScope.USER}:${key}`, user); - done(null, user); - }) - .catch((e) => done(e)); - } else { - return done(null, user); - } - }) - .catch((e) => { - console.log(e); - done({ msg: 'User not found' }); - }); - }) - .catch((e) => { - console.log(e); - done({ msg: 'Invalid token' }); - }); - } - ) - ); - - passport.serializeUser(function ( - { - id, - email, - email_verified, - roles: _roles, - provider, - firstname, - lastname, - isAuthorized, - isPublicBase, - token_version, - }, - done - ) { - const roles = (_roles || '') - .split(',') - .reduce((obj, role) => Object.assign(obj, { [role]: true }), {}); - if (roles.owner) { - roles.creator = true; - } - done(null, { - isAuthorized, - isPublicBase, - id, - email, - email_verified, - provider, - firstname, - lastname, - roles, - token_version, - }); - }); - - passport.deserializeUser(function (user, done) { - done(null, user); - }); - - passport.use( - new JwtStrategy( - { - secretOrKey: Noco.getConfig().auth.jwt.secret, - ...jwtOptions, - passReqToCallback: true, - ...Noco.getConfig().auth.jwt.options, - }, - async (req, jwtPayload, done) => { - // todo: improve this - if ( - req.ncProjectId && - jwtPayload.roles?.split(',').includes(OrgUserRoles.SUPER_ADMIN) - ) { - return User.getByEmail(jwtPayload?.email).then(async (user) => { - return done(null, { - ...user, - roles: `owner,creator,${OrgUserRoles.SUPER_ADMIN}`, - }); - }); - } - - const keyVals = [jwtPayload?.email]; - if (req.ncProjectId) { - keyVals.push(req.ncProjectId); - } - const key = keyVals.join('___'); - const cachedVal = await NocoCache.get( - `${CacheScope.USER}:${key}`, - CacheGetType.TYPE_OBJECT - ); - - if (cachedVal) { - if ( - !cachedVal.token_version || - !jwtPayload.token_version || - cachedVal.token_version !== jwtPayload.token_version - ) { - return done(new Error('Token Expired. Please login again.')); - } - return done(null, cachedVal); - } - - User.getByEmail(jwtPayload?.email) - .then(async (user) => { - if ( - !user.token_version || - !jwtPayload.token_version || - user.token_version !== jwtPayload.token_version - ) { - return done(new Error('Token Expired. Please login again.')); - } - if (req.ncProjectId) { - // this.xcMeta - // .metaGet(req.ncProjectId, null, 'nc_projects_users', { - // user_id: user?.id - // }) - - ProjectUser.get(req.ncProjectId, user.id) - .then(async (projectUser) => { - user.roles = projectUser?.roles || user.roles; - user.roles = - user.roles === 'owner' ? 'owner,creator' : user.roles; - // + (user.roles ? `,${user.roles}` : ''); - - await NocoCache.set(`${CacheScope.USER}:${key}`, user); - done(null, user); - }) - .catch((e) => done(e)); - } else { - // const roles = projectUser?.roles ? JSON.parse(projectUser.roles) : {guest: true}; - if (user) { - await NocoCache.set(`${CacheScope.USER}:${key}`, user); - return done(null, user); - } else { - return done(new Error('User not found')); - } - } - }) - .catch((err) => { - return done(err); - }); - } - ) - ); - - passport.use( - new PassportLocalStrategy( - { - usernameField: 'email', - session: false, - }, - async (email, password, done) => { - try { - const user = await User.getByEmail(email); - if (!user) { - return done({ msg: `Email ${email} is not registered!` }); - } - - if (!user.salt) { - return done({ - msg: `Please sign up with the invite token first or reset the password by clicking Forgot your password.`, - }); - } - - const hashedPassword = await promisify(bcrypt.hash)( - password, - user.salt - ); - if (user.password !== hashedPassword) { - return done({ msg: `Password not valid!` }); - } else { - return done(null, user); - } - } catch (e) { - done(e); - } - } - ) - ); - - passport.use( - 'baseView', - new CustomStrategy(async (req: any, callback) => { - let user; - if (req.headers['xc-shared-base-id']) { - // const cacheKey = `nc_shared_bases||${req.headers['xc-shared-base-id']}`; - - let sharedProject = null; - - if (!sharedProject) { - sharedProject = await Project.getByUuid( - req.headers['xc-shared-base-id'] - ); - } - user = { - roles: sharedProject?.roles, - }; - } - - callback(null, user); - }) - ); - - // mostly copied from older code - Plugin.getPluginByTitle('Google').then((googlePlugin) => { - if (googlePlugin && googlePlugin.input) { - const settings = JSON.parse(googlePlugin.input); - process.env.NC_GOOGLE_CLIENT_ID = settings.client_id; - process.env.NC_GOOGLE_CLIENT_SECRET = settings.client_secret; - } - - if ( - process.env.NC_GOOGLE_CLIENT_ID && - process.env.NC_GOOGLE_CLIENT_SECRET - ) { - const googleAuthParamsOrig = GoogleStrategy.prototype.authorizationParams; - GoogleStrategy.prototype.authorizationParams = (options: any) => { - const params = googleAuthParamsOrig.call(this, options); - - if (options.state) { - params.state = options.state; - } - - return params; - }; - - const clientConfig = { - clientID: process.env.NC_GOOGLE_CLIENT_ID, - clientSecret: process.env.NC_GOOGLE_CLIENT_SECRET, - // todo: update url - callbackURL: 'http://localhost:3000', - passReqToCallback: true, - }; - - const googleStrategy = new GoogleStrategy( - clientConfig, - async (req, _accessToken, _refreshToken, profile, done) => { - const email = profile.emails[0].value; - - User.getByEmail(email) - .then(async (user) => { - if (user) { - // if project id defined extract project level roles - if (req.ncProjectId) { - ProjectUser.get(req.ncProjectId, user.id) - .then(async (projectUser) => { - user.roles = projectUser?.roles || user.roles; - user.roles = - user.roles === 'owner' ? 'owner,creator' : user.roles; - // + (user.roles ? `,${user.roles}` : ''); - - done(null, user); - }) - .catch((e) => done(e)); - } else { - return done(null, user); - } - // if user not found create new user if allowed - // or return error - } else { - const salt = await promisify(bcrypt.genSalt)(10); - const user = await registerNewUserIfAllowed({ - firstname: null, - lastname: null, - email_verification_token: null, - email: profile.emails[0].value, - password: '', - salt, - }); - return done(null, user); - } - }) - .catch((err) => { - return done(err); - }); - } - ); - - passport.use(googleStrategy); - } - }); - - router.use(passport.initialize()); -} diff --git a/packages/nocodb/src/lib/controllers/userApi/ui/auth/emailVerify.ts b/packages/nocodb/src/lib/controllers/userApi/ui/auth/emailVerify.ts deleted file mode 100644 index f7412b12ba..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/ui/auth/emailVerify.ts +++ /dev/null @@ -1,70 +0,0 @@ -export default ` - - - NocoDB - Verify Email - - - - - - - -
- - - - - - Email verified successfully! - - - {{errMsg}} - - - - - - - -
- - - - - -`; diff --git a/packages/nocodb/src/lib/controllers/userApi/ui/auth/resetPassword.ts b/packages/nocodb/src/lib/controllers/userApi/ui/auth/resetPassword.ts deleted file mode 100644 index 514fc6d739..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/ui/auth/resetPassword.ts +++ /dev/null @@ -1,108 +0,0 @@ -export default ` - - - NocoDB - Reset Password - - - - - - - -
- - - - - - Password reset successful! - - - - - - -
- - - - - -`; diff --git a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/forgotPassword.ts b/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/forgotPassword.ts deleted file mode 100644 index afb2f5849a..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/forgotPassword.ts +++ /dev/null @@ -1,171 +0,0 @@ -export default ` - - - - - Simple Transactional Email - - - - - - - - - - - - - -`; diff --git a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/invite.ts b/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/invite.ts deleted file mode 100644 index fc81f9409e..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/invite.ts +++ /dev/null @@ -1,208 +0,0 @@ -export default ` - - - - - Simple Transactional Email - - - - - - - - - - - - - -`; diff --git a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/verify.ts b/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/verify.ts deleted file mode 100644 index 11702cc659..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/verify.ts +++ /dev/null @@ -1,207 +0,0 @@ -export default ` - - - - - Simple Transactional Email - - - - - - - - - - - - - -`; diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index 06da16fd10..f1a676997a 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -22,12 +22,12 @@ import hookApis from '../../controllers/hookController'; import pluginApis from '../../controllers/pluginController'; import gridViewColumnApis from '../../controllers/gridViewColumnController'; import kanbanViewApis from '../../controllers/kanbanViewController'; -import { userApis } from '../../controllers/userApi'; +import { userController } from '../../controllers/userController'; // import extractProjectIdAndAuthenticate from './helpers/extractProjectIdAndAuthenticate'; import utilApis from '../../controllers/utilController'; import projectUserApis from '../../controllers/projectUserController'; import sharedBaseApis from '../../controllers/sharedBaseController'; -import { initStrategies } from '../../controllers/userApi/initStrategies'; +import { initStrategies } from '../../controllers/userController/initStrategies'; import modelVisibilityApis from '../../controllers/modelVisibilityController'; import metaDiffApis from '../../controllers/metaDiffController'; import cacheApis from '../../controllers/cacheController'; @@ -51,9 +51,9 @@ import { Server, Socket } from 'socket.io'; import passport from 'passport'; import crypto from 'crypto'; -import swaggerApis from '../../controllers/swagger/swaggerApis'; -import importApis from '../../controllers/sync/importApis'; -import syncSourceApis from '../../controllers/sync/syncSourceApis'; +import swaggerApis from '../../controllers/swaggerController'; +import importApis from '../../controllers/syncController/importApis'; +import syncSourceApis from '../../controllers/syncController'; import mapViewApis from '../../controllers/mapViewController'; const clients: { [id: string]: Socket } = {}; @@ -108,7 +108,7 @@ export default function (router: Router, server) { router.use(kanbanViewApis); router.use(mapViewApis); - userApis(router); + userController(router); const io = new Server(server, { cors: { diff --git a/packages/nocodb/src/lib/services/attachmentService.ts b/packages/nocodb/src/lib/services/attachmentService.ts index f4a9bfdca1..1d1fc25f02 100644 --- a/packages/nocodb/src/lib/services/attachmentService.ts +++ b/packages/nocodb/src/lib/services/attachmentService.ts @@ -1,3 +1,6 @@ + + + // @ts-ignore import { Request, Response, Router } from 'express'; import { nanoid } from 'nanoid'; diff --git a/packages/nocodb/src/lib/services/projectUserService.ts b/packages/nocodb/src/lib/services/projectUserService.ts index 567568f488..2ff67913d4 100644 --- a/packages/nocodb/src/lib/services/projectUserService.ts +++ b/packages/nocodb/src/lib/services/projectUserService.ts @@ -282,7 +282,7 @@ export async function sendInviteEmail( ): Promise { try { const template = ( - await import('../meta/api/userApi/ui/emailTemplates/invite') + await import('./userService/ui/emailTemplates/invite') ).default; const emailAdapter = await NcPluginMgrv2.emailAdapter(); diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts index 883927583f..1ff534548f 100644 --- a/packages/nocodb/src/lib/services/tableService.ts +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -1,4 +1,3 @@ -// @ts-ignore import DOMPurify from 'isomorphic-dompurify'; import { AuditOperationSubTypes, @@ -28,6 +27,112 @@ import View from '../models/View'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import { T } from 'nc-help'; +export async function tableUpdate(param: { + tableId: any; + table: TableReqType & { project_id?: string }; + projectId?: string; +}) { + const model = await Model.get(param.tableId); + + const project = await Project.getWithInfo( + param.table.project_id || param.projectId + ); + const base = project.bases.find((b) => b.id === model.base_id); + + if (model.project_id !== project.id) { + NcError.badRequest('Model does not belong to project'); + } + + // if meta present update meta and return + // todo: allow user to update meta and other prop in single api call + if ('meta' in param.table) { + await Model.updateMeta(param.tableId, param.table.meta); + + return { msg: 'success' } + } + + if (!param.table.table_name) { + NcError.badRequest( + 'Missing table name `table_name` property in request body' + ); + } + + if (base.is_meta && project.prefix) { + if (!param.table.table_name.startsWith(project.prefix)) { + param.table.table_name = `${project.prefix}${param.table.table_name}`; + } + } + + param.table.table_name = DOMPurify.sanitize(param.table.table_name); + + // validate table name + if (/^\s+|\s+$/.test(param.table.table_name)) { + NcError.badRequest( + 'Leading or trailing whitespace not allowed in table names' + ); + } + + if ( + !(await Model.checkTitleAvailable({ + table_name: param.table.table_name, + project_id: project.id, + base_id: base.id, + })) + ) { + NcError.badRequest('Duplicate table name'); + } + + if (!param.table.title) { + param.table.title = getTableNameAlias( + param.table.table_name, + project.prefix, + base + ); + } + + if ( + !(await Model.checkAliasAvailable({ + title: param.table.title, + project_id: project.id, + base_id: base.id, + })) + ) { + NcError.badRequest('Duplicate table alias'); + } + + const sqlMgr = await ProjectMgrv2.getSqlMgr(project); + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + + let tableNameLengthLimit = 255; + const sqlClientType = sqlClient.knex.clientType(); + if (sqlClientType === 'mysql2' || sqlClientType === 'mysql') { + tableNameLengthLimit = 64; + } else if (sqlClientType === 'pg') { + tableNameLengthLimit = 63; + } else if (sqlClientType === 'mssql') { + tableNameLengthLimit = 128; + } + + if (param.table.table_name.length > tableNameLengthLimit) { + NcError.badRequest(`Table name exceeds ${tableNameLengthLimit} characters`); + } + + await Model.updateAliasAndTableName( + param.tableId, + param.table.title, + param.table.table_name + ); + + await sqlMgr.sqlOpPlus(base, 'tableRename', { + ...param.table, + tn: param.table.table_name, + tn_old: model.table_name, + }); + + T.emit('evt', { evt_type: 'table:updated' }); + return true; +} + export function reorderTable(param: { tableId: string; order: any }) { return Model.updateOrder(param.tableId, param.order); } @@ -100,7 +205,7 @@ export async function getTableWithAccessibleViews(param: { // todo: optimise const viewList = await xcVisibilityMetaGet(table.project_id, [table]); - //await View.list(req.params.tableId) + //await View.list(param.tableId) table.views = viewList.filter((table: any) => { return Object.keys(param.user?.roles).some( (role) => param.user?.roles[role] && !table.disabled[role] diff --git a/packages/nocodb/src/lib/services/userService/helpers.ts b/packages/nocodb/src/lib/services/userService/helpers.ts index 90e2d81dfc..387b51170e 100644 --- a/packages/nocodb/src/lib/services/userService/helpers.ts +++ b/packages/nocodb/src/lib/services/userService/helpers.ts @@ -1,7 +1,7 @@ import * as jwt from 'jsonwebtoken'; import crypto from 'crypto'; +import User from '../../models/User'; import { NcConfig } from '../../../interface/config'; -import { User } from '../../models'; export function genJwt(user: User, config: NcConfig) { return jwt.sign( From 6f494d0dbc6a9ef98abbc367d3c5fa7edd90462e Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Fri, 3 Mar 2023 09:29:36 +0530 Subject: [PATCH 38/64] test: disable redundant test logs Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- tests/playwright/tests/filters.spec.ts | 2 -- tests/playwright/tests/keyboardShortcuts.spec.ts | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/playwright/tests/filters.spec.ts b/tests/playwright/tests/filters.spec.ts index 2f5b64a43f..e399d85f80 100644 --- a/tests/playwright/tests/filters.spec.ts +++ b/tests/playwright/tests/filters.spec.ts @@ -86,8 +86,6 @@ async function verifyFilter(param: { return; } - console.log(`Verifying filter: ${param.opType} ${param.opSubType}`); - await toolbar.clickFilter(); await toolbar.filter.add({ columnTitle: param.column, diff --git a/tests/playwright/tests/keyboardShortcuts.spec.ts b/tests/playwright/tests/keyboardShortcuts.spec.ts index 5026ccbb5d..2c027f542d 100644 --- a/tests/playwright/tests/keyboardShortcuts.spec.ts +++ b/tests/playwright/tests/keyboardShortcuts.spec.ts @@ -232,8 +232,7 @@ test.describe('Verify shortcuts', () => { }); await api.dbTableRow.bulkCreate('noco', context.project.id, table.id, [record]); - const records = await api.dbTableRow.list('noco', context.project.id, table.id, { limit: 1 }); - console.log(records); + await api.dbTableRow.list('noco', context.project.id, table.id, { limit: 1 }); } catch (e) { console.error(e); } From 2910fcc1b164ec5c7cce3950c7dbe286f89e5ffb Mon Sep 17 00:00:00 2001 From: navi Date: Fri, 3 Mar 2023 13:07:28 +0000 Subject: [PATCH 39/64] New translations en.json (Danish) --- packages/nc-gui/lang/da.json | 44 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/nc-gui/lang/da.json b/packages/nc-gui/lang/da.json index e67b4258f8..5a769dccb7 100644 --- a/packages/nc-gui/lang/da.json +++ b/packages/nc-gui/lang/da.json @@ -1,31 +1,31 @@ { "general": { "home": "Hjem", - "load": "belastning", - "open": "Åben", - "close": "Tæt", + "load": "Indlæs", + "open": "Åbn", + "close": "Luk", "yes": "Ja", - "no": "Ingen", + "no": "Nej", "ok": "Okay", "and": "Og", "or": "Eller", - "add": "Tilføje", - "edit": "Redigere", - "remove": "Fjerne", - "save": "Gemme", - "cancel": "Afbestille", + "add": "Tilføj", + "edit": "Redigér", + "remove": "Fjern", + "save": "Gem", + "cancel": "Fortryd", "submit": "Indsend", - "create": "skab", + "create": "Opret", "duplicate": "Duplikat", - "insert": "Indsættes", - "delete": "Delete.", - "update": "UPDATE.", + "insert": "Indsæt", + "delete": "Slet", + "update": "Opdatér", "rename": "Omdøb", - "reload": "Genindlæsning", + "reload": "Genindlæs", "reset": "Nulstil", - "install": "Installere", - "show": "At vise", - "hide": "Skjule", + "install": "Installer", + "show": "Vis", + "hide": "Skjul", "showAll": "Vis alt", "hideAll": "Gem alt", "showMore": "Vis mere", @@ -100,7 +100,7 @@ "form": "Formular", "kanban": "Kanban.", "calendar": "Kalender", - "map": "Map" + "map": "Kort" }, "user": "Bruger", "users": "Brugere", @@ -209,7 +209,7 @@ "advancedSettings": "Avancerede indstillinger", "codeSnippet": "Kodeuddrag", "keyboardShortcut": "Tastaturgenveje", - "generateRandomName": "Generate Random Name" + "generateRandomName": "Generér Tilfældigt Navn" }, "labels": { "createdBy": "Oprettet af", @@ -257,7 +257,7 @@ "barcodeFormat": "Stregkodeformat", "qrCodeValueTooLong": "For mange tegn til en QR-kode", "barcodeValueTooLong": "For mange tegn til en stregkode", - "yourLocation": "Your Location", + "yourLocation": "Din Placering", "lng": "Lng", "lat": "Lat", "aggregateFunction": "Aggregate Function.", @@ -387,14 +387,14 @@ "renameTable": "Bord omdøb", "deleteTable": "TABEL DELETE.", "addField": "Tilføj nyt felt til denne tabel", - "setDisplay": "Set as Display value", + "setDisplay": "Sæt som visningsværdi", "addRow": "Tilføj ny række", "saveRow": "Gem ro", "saveAndExit": "Gem og afslutning", "saveAndStay": "Gem og bliv", "insertRow": "Indsæt ny række", "deleteRow": "DELETE ROW.", - "duplicateRow": "Duplicate Row", + "duplicateRow": "Dupliker Række", "deleteSelectedRow": "Slet de valgte rækker", "importExcel": "Import Excel.", "importCSV": "Import CSV.", From b7da5eaf40e9b6651891cf19a34cbf4d470f496d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Mar 2023 10:35:25 +0530 Subject: [PATCH 40/64] refactor: move ajv validator from express middleware to service layer Signed-off-by: Pranav C --- .../src/lib/controllers/apiTokenController.ts | 36 +- .../src/lib/controllers/auditController.ts | 13 +- .../src/lib/controllers/baseController.ts | 100 ++---- .../src/lib/controllers/columnController.ts | 2 - .../dataApis/dataAliasExportApis.ts | 6 +- .../src/lib/controllers/filterController.ts | 4 - .../controllers/formViewColumnController.ts | 2 - .../src/lib/controllers/formViewController.ts | 50 +-- .../lib/controllers/galleryViewController.ts | 3 - .../controllers/gridViewColumnController.ts | 2 - .../src/lib/controllers/gridViewController.ts | 35 +- .../src/lib/controllers/hookController.ts | 4 - .../lib/controllers/hookFilterController.ts | 3 - .../lib/controllers/kanbanViewController.ts | 5 +- .../controllers/modelVisibilityController.ts | 2 - .../lib/controllers/orgLicenseController.ts | 2 - .../src/lib/controllers/orgTokenController.ts | 56 +--- .../src/lib/controllers/orgUserController.ts | 3 - .../src/lib/controllers/pluginController.ts | 4 - .../src/lib/controllers/projectController.ts | 2 - .../lib/controllers/projectUserController.ts | 316 ++---------------- .../lib/controllers/sharedBaseController.ts | 97 ++---- .../src/lib/controllers/sortController.ts | 3 - .../src/lib/controllers/tableController.ts | 117 ++++++- .../lib/controllers/userController/index.ts | 22 +- .../src/lib/meta/api/helpers/apiHelpers.ts | 6 +- packages/nocodb/src/lib/models/Hook.ts | 164 ++++----- .../src/lib/services/apiTokenService.ts | 6 + .../nocodb/src/lib/services/auditService.ts | 11 + .../nocodb/src/lib/services/baseService.ts | 6 +- .../nocodb/src/lib/services/columnService.ts | 3 + .../src/lib/services/dataService/index.ts | 2 +- .../nocodb/src/lib/services/filterService.ts | 7 + .../src/lib/services/formViewColumnService.ts | 6 + .../src/lib/services/formViewService.ts | 5 + .../src/lib/services/galleryViewService.ts | 5 + .../src/lib/services/gridViewColumnService.ts | 3 + .../src/lib/services/gridViewService.ts | 4 + .../src/lib/services/hookFilterService.ts | 5 + .../nocodb/src/lib/services/hookService.ts | 10 + .../src/lib/services/kanbanViewService.ts | 10 +- .../lib/services/modelVisibilityService.ts | 7 +- .../src/lib/services/orgLicenseService.ts | 3 + .../src/lib/services/orgTokenService.ts | 6 + .../nocodb/src/lib/services/orgUserService.ts | 5 + .../nocodb/src/lib/services/pluginService.ts | 5 + .../nocodb/src/lib/services/projectService.ts | 4 +- .../src/lib/services/projectUserService.ts | 11 + .../src/lib/services/sharedBaseService.ts | 5 + .../nocodb/src/lib/services/sortService.ts | 5 + .../nocodb/src/lib/services/tableService.ts | 21 +- .../src/lib/services/userService/index.ts | 16 + 52 files changed, 513 insertions(+), 717 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/apiTokenController.ts b/packages/nocodb/src/lib/controllers/apiTokenController.ts index 33e940c0e5..d472c9a800 100644 --- a/packages/nocodb/src/lib/controllers/apiTokenController.ts +++ b/packages/nocodb/src/lib/controllers/apiTokenController.ts @@ -1,31 +1,28 @@ import { Request, Response, Router } from 'express'; -import { OrgUserRoles } from 'nocodb-sdk'; -import { Tele } from 'nc-help'; -import { NcError } from '../meta/helpers/catchError'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import ApiToken from '../models/ApiToken'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; +import { apiTokenService } from '../services'; export async function apiTokenList(req: Request, res: Response) { - res.json(await ApiToken.list(req['user'].id)); + res.json(await apiTokenService.apiTokenList({ userId: req['user'].id })); } + export async function apiTokenCreate(req: Request, res: Response) { - Tele.emit('evt', { evt_type: 'apiToken:created' }); - res.json(await ApiToken.insert({ ...req.body, fk_user_id: req['user'].id })); + res.json( + await apiTokenService.apiTokenCreate({ + tokenBody: req.body, + userId: req['user'].id, + }) + ); } -export async function apiTokenDelete(req: Request, res: Response) { - const apiToken = await ApiToken.getByToken(req.params.token); - if ( - !req['user'].roles.includes(OrgUserRoles.SUPER_ADMIN) && - apiToken.fk_user_id !== req['user'].id - ) { - NcError.notFound('Token not found'); - } - Tele.emit('evt', { evt_type: 'apiToken:deleted' }); - // todo: verify token belongs to the user - res.json(await ApiToken.delete(req.params.token)); +export async function apiTokenDelete(req: Request, res: Response) { + res.json( + await apiTokenService.apiTokenDelete({ + token: req.params.token, + user: req['user'], + }) + ); } // todo: add reset token api to regenerate token @@ -41,7 +38,6 @@ router.get( router.post( '/api/v1/db/meta/projects/:projectId/api-tokens', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/ApiTokenReq'), ncMetaAclMw(apiTokenCreate, 'apiTokenCreate') ); router.delete( diff --git a/packages/nocodb/src/lib/controllers/auditController.ts b/packages/nocodb/src/lib/controllers/auditController.ts index ee015f638a..366fc70529 100644 --- a/packages/nocodb/src/lib/controllers/auditController.ts +++ b/packages/nocodb/src/lib/controllers/auditController.ts @@ -1,18 +1,17 @@ import { Request, Response, Router } from 'express'; import Audit from '../models/Audit'; -import { AuditOperationTypes } from 'nocodb-sdk'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; - -import { getAjvValidatorMw } from '../meta/api/helpers'; import { auditService } from '../services'; export async function commentRow(req: Request, res) { res.json( - await Audit.insert({ - ...req.body, + await auditService.commentRow({ + rowId: req.params.rowId, user: (req as any).user, - op_type: AuditOperationTypes.COMMENT, + body: { + ...req.body, + }, }) ); } @@ -60,12 +59,10 @@ router.get( ); router.post( '/api/v1/db/meta/audits/comments', - getAjvValidatorMw('swagger.json#/components/schemas/CommentReq'), ncMetaAclMw(commentRow, 'commentRow') ); router.post( '/api/v1/db/meta/audits/rows/:rowId/update', - getAjvValidatorMw('swagger.json#/components/schemas/AuditRowUpdateReq'), ncMetaAclMw(auditRowUpdate, 'auditRowUpdate') ); router.get( diff --git a/packages/nocodb/src/lib/controllers/baseController.ts b/packages/nocodb/src/lib/controllers/baseController.ts index 1ac4d3e3ba..f35deb5069 100644 --- a/packages/nocodb/src/lib/controllers/baseController.ts +++ b/packages/nocodb/src/lib/controllers/baseController.ts @@ -1,104 +1,66 @@ import { Request, Response } from 'express'; -import Project from '../models/Project'; import { BaseListType } from 'nocodb-sdk'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import { syncBaseMigration } from '../meta/helpers/syncMigration'; import Base from '../models/Base'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { Tele } from 'nc-help'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw, populateMeta } from '../meta/api/helpers'; -export async function baseGet( - req: Request, - res: Response -) { - const base = await Base.get(req.params.baseId); +import { baseService } from '../services'; - base.config = base.getConnectionConfig(); +async function baseGet(req: Request, res: Response) { + const base = await baseService.baseGetWithConfig({ + baseId: req.params.baseId, + }); res.json(base); } -export async function baseUpdate( - req: Request, - res: Response -) { - const baseBody = req.body; - const project = await Project.getWithInfo(req.params.projectId); - const base = await Base.updateBase(req.params.baseId, { - ...baseBody, - type: baseBody.config?.client, - projectId: project.id, - id: req.params.baseId, - }); - - delete base.config; - - Tele.emit('evt', { - evt_type: 'base:updated', +async function baseUpdate(req: Request, res: Response) { + const base = await baseService.baseUpdate({ + baseId: req.params.baseId, + base: req.body, + projectId: req.params.projectId, }); - res.json(base); } -export async function baseList( +async function baseList( req: Request, - res: Response, - next + res: Response ) { - try { - const bases = await Base.list({ projectId: req.params.projectId }); + const bases = await baseService.baseList({ + projectId: req.params.projectId, + }); - res // todo: pagination - .json({ - bases: new PagedResponseImpl(bases, { - count: bases.length, - limit: bases.length, - }), - }); - } catch (e) { - console.log(e); - next(e); - } + res // todo: pagination + .json({ + bases: new PagedResponseImpl(bases, { + count: bases.length, + limit: bases.length, + }), + }); } export async function baseDelete( req: Request, res: Response ) { - const base = await Base.get(req.params.baseId); - const result = await base.delete(); - Tele.emit('evt', { evt_type: 'base:deleted' }); + const result = await baseService.baseDelete({ + baseId: req.params.baseId, + }); res.json(result); } async function baseCreate(req: Request, res) { - // type | base | projectId - const baseBody = req.body; - const project = await Project.getWithInfo(req.params.projectId); - const base = await Base.createBase({ - ...baseBody, - type: baseBody.config?.client, - projectId: project.id, - }); - - await syncBaseMigration(project, base); - - const info = await populateMeta(base, project); - - Tele.emit('evt_api_created', info); - - delete base.config; - - Tele.emit('evt', { - evt_type: 'base:created', + const base = await baseService.baseCreate({ + projectId: req.params.projectId, + base: req.body, }); res.json(base); } -export default (router) => { +const initRoutes = (router) => { router.get( '/api/v1/db/meta/projects/:projectId/bases/:baseId', metaApiMetrics, @@ -107,7 +69,6 @@ export default (router) => { router.patch( '/api/v1/db/meta/projects/:projectId/bases/:baseId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/BaseReq'), ncMetaAclMw(baseUpdate, 'baseUpdate') ); router.delete( @@ -118,7 +79,6 @@ export default (router) => { router.post( '/api/v1/db/meta/projects/:projectId/bases', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/BaseReq'), ncMetaAclMw(baseCreate, 'baseCreate') ); router.get( @@ -127,3 +87,5 @@ export default (router) => { ncMetaAclMw(baseList, 'baseList') ); }; + +export default initRoutes; diff --git a/packages/nocodb/src/lib/controllers/columnController.ts b/packages/nocodb/src/lib/controllers/columnController.ts index 61f2e0139d..8f6b34e12d 100644 --- a/packages/nocodb/src/lib/controllers/columnController.ts +++ b/packages/nocodb/src/lib/controllers/columnController.ts @@ -1,6 +1,5 @@ import { Request, Response, Router } from 'express'; import { ColumnReqType, TableType, UITypes } from 'nocodb-sdk'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { columnService } from '../services'; @@ -45,7 +44,6 @@ const router = Router({ mergeParams: true }); router.post( '/api/v1/db/meta/tables/:tableId/columns/', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/ColumnReq'), ncMetaAclMw(columnAdd, 'columnAdd') ); diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts index f0f145f596..c628e31814 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts @@ -15,7 +15,11 @@ async function excelDataExport(req: Request, res: Response) { if (!targetView) { targetView = await View.getDefaultView(model.id); } - const { offset, elapsed, data } = await extractXlsxData({ view: targetView, query: req.query, siteUrl: (req as any).ncSiteUrl }); + const { offset, elapsed, data } = await extractXlsxData({ + view: targetView, + query: req.query, + siteUrl: (req as any).ncSiteUrl, + }); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, data, targetView.title); const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); diff --git a/packages/nocodb/src/lib/controllers/filterController.ts b/packages/nocodb/src/lib/controllers/filterController.ts index d83f3e22b7..bf48e30765 100644 --- a/packages/nocodb/src/lib/controllers/filterController.ts +++ b/packages/nocodb/src/lib/controllers/filterController.ts @@ -1,6 +1,5 @@ import { Request, Response, Router } from 'express'; import { FilterReqType } from 'nocodb-sdk'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; @@ -80,7 +79,6 @@ router.get( router.post( '/api/v1/db/meta/views/:viewId/filters', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/FilterReq'), ncMetaAclMw(filterCreate, 'filterCreate') ); @@ -91,7 +89,6 @@ router.get( router.post( '/api/v1/db/meta/hooks/:hookId/filters', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/FilterReq'), ncMetaAclMw(hookFilterCreate, 'filterCreate') ); @@ -103,7 +100,6 @@ router.get( router.patch( '/api/v1/db/meta/filters/:filterId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/FilterReq'), ncMetaAclMw(filterUpdate, 'filterUpdate') ); router.delete( diff --git a/packages/nocodb/src/lib/controllers/formViewColumnController.ts b/packages/nocodb/src/lib/controllers/formViewColumnController.ts index 532b5eb82a..8b13309411 100644 --- a/packages/nocodb/src/lib/controllers/formViewColumnController.ts +++ b/packages/nocodb/src/lib/controllers/formViewColumnController.ts @@ -1,7 +1,6 @@ import { Request, Response, Router } from 'express'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { formViewColumnService } from '../services'; export async function columnUpdate(req: Request, res: Response) { @@ -17,7 +16,6 @@ const router = Router({ mergeParams: true }); router.patch( '/api/v1/db/meta/form-columns/:formViewColumnId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/FormColumnReq'), ncMetaAclMw(columnUpdate, 'columnUpdate') ); export default router; diff --git a/packages/nocodb/src/lib/controllers/formViewController.ts b/packages/nocodb/src/lib/controllers/formViewController.ts index 02f8bc8e44..2bc61fd58d 100644 --- a/packages/nocodb/src/lib/controllers/formViewController.ts +++ b/packages/nocodb/src/lib/controllers/formViewController.ts @@ -1,50 +1,37 @@ import { Request, Response, Router } from 'express'; -// @ts-ignore -import Model from '../models/Model'; -import { Tele } from 'nc-help'; -// @ts-ignore -import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import { FormType, ViewTypes } from 'nocodb-sdk'; -// @ts-ignore -import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; -// @ts-ignore -import Project from '../models/Project'; -import View from '../models/View'; -import FormView from '../models/FormView'; +import { FormType } from 'nocodb-sdk'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; +import { formViewService } from '../services'; -// @ts-ignore export async function formViewGet(req: Request, res: Response) { - const formViewData = await FormView.getWithInfo(req.params.formViewId); + const formViewData = await formViewService.formViewGet({ + formViewId: req.params.formViewId, + }); res.json(formViewData); } export async function formViewCreate(req: Request, res) { - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'form' }); - const view = await View.insert({ - ...req.body, - // todo: sanitize - fk_model_id: req.params.tableId, - type: ViewTypes.FORM, + const view = await formViewService.formViewCreate({ + body: req.body, + tableId: req.params.tableId, }); res.json(view); } -// @ts-ignore + export async function formViewUpdate(req, res) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'grid' }); - res.json(await FormView.update(req.params.formViewId, req.body)); + res.json( + await formViewService.formViewUpdate({ + formViewId: req.params.formViewId, + body: req.body, + }) + ); } -// @ts-ignore -export async function formViewDelete(req: Request, res: Response, next) {} - const router = Router({ mergeParams: true }); router.post( '/api/v1/db/meta/tables/:tableId/forms', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/FormCreateReq'), ncMetaAclMw(formViewCreate, 'formViewCreate') ); router.get( @@ -55,12 +42,7 @@ router.get( router.patch( '/api/v1/db/meta/forms/:formViewId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/FormReq'), ncMetaAclMw(formViewUpdate, 'formViewUpdate') ); -router.delete( - '/api/v1/db/meta/forms/:formViewId', - metaApiMetrics, - ncMetaAclMw(formViewDelete, 'formViewDelete') -); + export default router; diff --git a/packages/nocodb/src/lib/controllers/galleryViewController.ts b/packages/nocodb/src/lib/controllers/galleryViewController.ts index 947ea6b65a..f54dc901cd 100644 --- a/packages/nocodb/src/lib/controllers/galleryViewController.ts +++ b/packages/nocodb/src/lib/controllers/galleryViewController.ts @@ -2,7 +2,6 @@ import { Request, Response, Router } from 'express'; import { GalleryType } from 'nocodb-sdk'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { galleryViewService } from '../services'; export async function galleryViewGet(req: Request, res: Response) { @@ -35,13 +34,11 @@ const router = Router({ mergeParams: true }); router.post( '/api/v1/db/meta/tables/:tableId/galleries', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/GalleryReq'), ncMetaAclMw(galleryViewCreate, 'galleryViewCreate') ); router.patch( '/api/v1/db/meta/galleries/:galleryViewId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/GalleryReq'), ncMetaAclMw(galleryViewUpdate, 'galleryViewUpdate') ); router.get( diff --git a/packages/nocodb/src/lib/controllers/gridViewColumnController.ts b/packages/nocodb/src/lib/controllers/gridViewColumnController.ts index 3b535cce9e..1ce0a41be7 100644 --- a/packages/nocodb/src/lib/controllers/gridViewColumnController.ts +++ b/packages/nocodb/src/lib/controllers/gridViewColumnController.ts @@ -1,7 +1,6 @@ import { Request, Response, Router } from 'express'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { gridViewColumnService } from '../services'; export async function columnList(req: Request, res: Response) { @@ -30,7 +29,6 @@ router.get( router.patch( '/api/v1/db/meta/grid-columns/:gridViewColumnId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/GridColumnReq'), ncMetaAclMw(gridColumnUpdate, 'gridColumnUpdate') ); export default router; diff --git a/packages/nocodb/src/lib/controllers/gridViewController.ts b/packages/nocodb/src/lib/controllers/gridViewController.ts index 2038717232..865ccecdb6 100644 --- a/packages/nocodb/src/lib/controllers/gridViewController.ts +++ b/packages/nocodb/src/lib/controllers/gridViewController.ts @@ -1,42 +1,29 @@ import { Request, Router } from 'express'; -// @ts-ignore -import Model from '../models/Model'; -import { Tele } from 'nc-help'; -// @ts-ignore -import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import { ViewTypes } from 'nocodb-sdk'; -// @ts-ignore -import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; -// @ts-ignore -import Project from '../models/Project'; -import View from '../models/View'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import GridView from '../models/GridView'; -import { getAjvValidatorMw } from '../meta/api/helpers'; +import { gridViewService } from '../services'; -// @ts-ignore -export async function gridViewCreate(req: Request, res) { - const view = await View.insert({ - ...req.body, - // todo: sanitize - fk_model_id: req.params.tableId, - type: ViewTypes.GRID, +export async function gridViewCreate(req: Request, res) { + const view = await gridViewService.gridViewCreate({ + grid: req.body, + tableId: req.params.tableId, }); - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'grid' }); res.json(view); } export async function gridViewUpdate(req, res) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'grid' }); - res.json(await GridView.update(req.params.viewId, req.body)); + res.json( + await gridViewService.gridViewUpdate({ + viewId: req.params.viewId, + grid: req.body, + }) + ); } const router = Router({ mergeParams: true }); router.post( '/api/v1/db/meta/tables/:tableId/grids/', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/GridReq'), ncMetaAclMw(gridViewCreate, 'gridViewCreate') ); router.patch( diff --git a/packages/nocodb/src/lib/controllers/hookController.ts b/packages/nocodb/src/lib/controllers/hookController.ts index c17280a9e9..c75e2b82d1 100644 --- a/packages/nocodb/src/lib/controllers/hookController.ts +++ b/packages/nocodb/src/lib/controllers/hookController.ts @@ -4,7 +4,6 @@ import { HookListType, HookType } from 'nocodb-sdk'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { hookService } from '../services'; export async function hookList( @@ -74,13 +73,11 @@ router.get( router.post( '/api/v1/db/meta/tables/:tableId/hooks/test', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/HookTestReq'), ncMetaAclMw(hookTest, 'hookTest') ); router.post( '/api/v1/db/meta/tables/:tableId/hooks', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/HookReq'), ncMetaAclMw(hookCreate, 'hookCreate') ); router.delete( @@ -91,7 +88,6 @@ router.delete( router.patch( '/api/v1/db/meta/hooks/:hookId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/HookReq'), ncMetaAclMw(hookUpdate, 'hookUpdate') ); router.get( diff --git a/packages/nocodb/src/lib/controllers/hookFilterController.ts b/packages/nocodb/src/lib/controllers/hookFilterController.ts index 71e3977802..c83a754d05 100644 --- a/packages/nocodb/src/lib/controllers/hookFilterController.ts +++ b/packages/nocodb/src/lib/controllers/hookFilterController.ts @@ -2,7 +2,6 @@ import { Request, Response, Router } from 'express'; import { T } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { hookFilterService } from '../services'; export async function filterGet(req: Request, res: Response) { @@ -66,7 +65,6 @@ router.get( router.post( '/hooks/:hookId/filters/', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/FilterReq'), ncMetaAclMw(filterCreate, 'filterCreate') ); router.get( @@ -77,7 +75,6 @@ router.get( router.patch( '/hooks/:hookId/filters/:filterId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/FilterReq'), ncMetaAclMw(filterUpdate, 'filterUpdate') ); router.delete( diff --git a/packages/nocodb/src/lib/controllers/kanbanViewController.ts b/packages/nocodb/src/lib/controllers/kanbanViewController.ts index 7ea705e6bb..bf36b43b90 100644 --- a/packages/nocodb/src/lib/controllers/kanbanViewController.ts +++ b/packages/nocodb/src/lib/controllers/kanbanViewController.ts @@ -5,7 +5,8 @@ import KanbanView from '../models/KanbanView'; import { T } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; + +// todo: map to service export async function kanbanViewGet(req: Request, res: Response) { res.json(await KanbanView.get(req.params.kanbanViewId)); @@ -32,13 +33,11 @@ const router = Router({ mergeParams: true }); router.post( '/api/v1/db/meta/tables/:tableId/kanbans', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/KanbanReq'), ncMetaAclMw(kanbanViewCreate, 'kanbanViewCreate') ); router.patch( '/api/v1/db/meta/kanbans/:kanbanViewId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/KanbanUpdateReq'), ncMetaAclMw(kanbanViewUpdate, 'kanbanViewUpdate') ); router.get( diff --git a/packages/nocodb/src/lib/controllers/modelVisibilityController.ts b/packages/nocodb/src/lib/controllers/modelVisibilityController.ts index 3f298c0d65..6af9a71506 100644 --- a/packages/nocodb/src/lib/controllers/modelVisibilityController.ts +++ b/packages/nocodb/src/lib/controllers/modelVisibilityController.ts @@ -1,7 +1,6 @@ import { Router } from 'express'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { modelVisibilityService } from '../services'; async function xcVisibilityMetaSetAll(req, res) { @@ -30,7 +29,6 @@ router.get( router.post( '/api/v1/db/meta/projects/:projectId/visibility-rules', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/VisibilityRuleReq'), ncMetaAclMw(xcVisibilityMetaSetAll, 'modelVisibilitySet') ); export default router; diff --git a/packages/nocodb/src/lib/controllers/orgLicenseController.ts b/packages/nocodb/src/lib/controllers/orgLicenseController.ts index de6d839537..23e5948ba1 100644 --- a/packages/nocodb/src/lib/controllers/orgLicenseController.ts +++ b/packages/nocodb/src/lib/controllers/orgLicenseController.ts @@ -2,7 +2,6 @@ import { Router } from 'express'; import { OrgUserRoles } from 'nocodb-sdk'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { orgLicenseService } from '../services'; async function licenseGet(_req, res) { @@ -26,7 +25,6 @@ router.get( router.post( '/api/v1/license', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/LicenseReq'), ncMetaAclMw(licenseSet, 'licenseSet', { allowedRoles: [OrgUserRoles.SUPER_ADMIN], blockApiTokenAccess: true, diff --git a/packages/nocodb/src/lib/controllers/orgTokenController.ts b/packages/nocodb/src/lib/controllers/orgTokenController.ts index 1e32de65c7..293ccd8c38 100644 --- a/packages/nocodb/src/lib/controllers/orgTokenController.ts +++ b/packages/nocodb/src/lib/controllers/orgTokenController.ts @@ -1,56 +1,35 @@ import { Request, Response, Router } from 'express'; -import { OrgUserRoles } from 'nocodb-sdk'; -import ApiToken from '../models/ApiToken'; -import { Tele } from 'nc-help'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { NcError } from '../meta/helpers/catchError'; import getHandler from '../meta/helpers/getHandler'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import { apiTokenListEE } from '../meta/api/ee/orgTokenApis'; -import { getAjvValidatorMw } from '../meta/api/helpers'; +import { orgTokenService } from '../services'; async function apiTokenList(req, res) { - const fk_user_id = req.user.id; - let includeUnmappedToken = false; - if (req['user'].roles.includes(OrgUserRoles.SUPER_ADMIN)) { - includeUnmappedToken = true; - } - res.json( - new PagedResponseImpl( - await ApiToken.listWithCreatedBy({ - ...req.query, - fk_user_id, - includeUnmappedToken, - }), - { - ...req.query, - count: await ApiToken.count({ - includeUnmappedToken, - fk_user_id, - }), - } - ) + await orgTokenService.apiTokenList({ + query: req.query, + user: req['user'], + }) ); } export async function apiTokenCreate(req: Request, res: Response) { - Tele.emit('evt', { evt_type: 'org:apiToken:created' }); - res.json(await ApiToken.insert({ ...req.body, fk_user_id: req['user'].id })); + res.json( + await orgTokenService.apiTokenCreate({ + apiToken: req.body, + user: req['user'], + }) + ); } export async function apiTokenDelete(req: Request, res: Response) { - const fk_user_id = req['user'].id; - const apiToken = await ApiToken.getByToken(req.params.token); - if ( - !req['user'].roles.includes(OrgUserRoles.SUPER_ADMIN) && - apiToken.fk_user_id !== fk_user_id - ) { - NcError.notFound('Token not found'); - } - Tele.emit('evt', { evt_type: 'org:apiToken:deleted' }); - res.json(await ApiToken.delete(req.params.token)); + res.json( + await orgTokenService.apiTokenDelete({ + token: req.params.token, + user: req['user'], + }) + ); } const router = Router({ mergeParams: true }); @@ -66,7 +45,6 @@ router.get( router.post( '/api/v1/tokens', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/ApiTokenReq'), ncMetaAclMw(apiTokenCreate, 'apiTokenCreate', { // allowedRoles: [OrgUserRoles.SUPER], blockApiTokenAccess: true, diff --git a/packages/nocodb/src/lib/controllers/orgUserController.ts b/packages/nocodb/src/lib/controllers/orgUserController.ts index c89e031124..e2e6df488a 100644 --- a/packages/nocodb/src/lib/controllers/orgUserController.ts +++ b/packages/nocodb/src/lib/controllers/orgUserController.ts @@ -2,7 +2,6 @@ import { Router } from 'express'; import { OrgUserRoles } from 'nocodb-sdk'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { orgUserService } from '../services'; async function userList(req, res) { @@ -87,7 +86,6 @@ router.get( router.patch( '/api/v1/users/:userId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/OrgUserReq'), ncMetaAclMw(userUpdate, 'userUpdate', { allowedRoles: [OrgUserRoles.SUPER_ADMIN], blockApiTokenAccess: true, @@ -104,7 +102,6 @@ router.delete( router.post( '/api/v1/users', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/OrgUserReq'), ncMetaAclMw(userAdd, 'userAdd', { allowedRoles: [OrgUserRoles.SUPER_ADMIN], blockApiTokenAccess: true, diff --git a/packages/nocodb/src/lib/controllers/pluginController.ts b/packages/nocodb/src/lib/controllers/pluginController.ts index 156fce97db..8d3e1b6943 100644 --- a/packages/nocodb/src/lib/controllers/pluginController.ts +++ b/packages/nocodb/src/lib/controllers/pluginController.ts @@ -3,7 +3,6 @@ import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import { PluginType } from 'nocodb-sdk'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { pluginService } from '../services'; export async function pluginList(_req: Request, res: Response) { @@ -42,8 +41,6 @@ router.get( router.post( '/api/v1/db/meta/plugins/test', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/PluginTestReq'), - ncMetaAclMw(pluginTest, 'pluginTest') ); router.get( @@ -54,7 +51,6 @@ router.get( router.patch( '/api/v1/db/meta/plugins/:pluginId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/PluginReq'), ncMetaAclMw(pluginUpdate, 'pluginUpdate') ); router.get( diff --git a/packages/nocodb/src/lib/controllers/projectController.ts b/packages/nocodb/src/lib/controllers/projectController.ts index 1be862a7e4..18d761efb1 100644 --- a/packages/nocodb/src/lib/controllers/projectController.ts +++ b/packages/nocodb/src/lib/controllers/projectController.ts @@ -11,7 +11,6 @@ import ProjectUser from '../models/ProjectUser'; import Noco from '../Noco'; import isDocker from 'is-docker'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import Filter from '../models/Filter'; import { projectService } from '../services'; @@ -155,7 +154,6 @@ export default (router) => { router.post( '/api/v1/db/meta/projects', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/ProjectReq'), ncMetaAclMw(projectCreate, 'projectCreate') ); router.get( diff --git a/packages/nocodb/src/lib/controllers/projectUserController.ts b/packages/nocodb/src/lib/controllers/projectUserController.ts index 6be2ff985c..f2bace15a4 100644 --- a/packages/nocodb/src/lib/controllers/projectUserController.ts +++ b/packages/nocodb/src/lib/controllers/projectUserController.ts @@ -1,305 +1,59 @@ -import { OrgUserRoles } from 'nocodb-sdk'; -import { Tele } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { Router } from 'express'; -import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import ProjectUser from '../models/ProjectUser'; -import validator from 'validator'; -import { NcError } from '../meta/helpers/catchError'; -import { v4 as uuidv4 } from 'uuid'; -import User from '../models/User'; -import Audit from '../models/Audit'; -import NocoCache from '../cache/NocoCache'; -import { CacheGetType, CacheScope, MetaTable } from '../utils/globals'; -import * as ejs from 'ejs'; -import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; -import Noco from '../Noco'; -import { PluginCategory } from 'nocodb-sdk'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { randomTokenString } from '../meta/helpers/stringHelpers'; -import { getAjvValidatorMw } from '../meta/api/helpers'; +import { projectUserService } from '../services'; async function userList(req, res) { res.json({ - users: new PagedResponseImpl( - await ProjectUser.getUsersList({ - ...req.query, - project_id: req.params.projectId, - }), - { - ...req.query, - count: await ProjectUser.getUsersCount(req.query), - } - ), + users: await projectUserService.userList({ + projectId: req.params.projectId, + query: req.query, + }), }); } -async function userInvite(req, res, next): Promise { - const emails = (req.body.email || '') - .toLowerCase() - .split(/\s*,\s*/) - .map((v) => v.trim()); - - // check for invalid emails - const invalidEmails = emails.filter((v) => !validator.isEmail(v)); - if (!emails.length) { - return NcError.badRequest('Invalid email address'); - } - if (invalidEmails.length) { - NcError.badRequest('Invalid email address : ' + invalidEmails.join(', ')); - } - - const invite_token = uuidv4(); - const error = []; - - for (const email of emails) { - // add user to project if user already exist - const user = await User.getByEmail(email); - - if (user) { - // check if this user has been added to this project - const projectUser = await ProjectUser.get(req.params.projectId, user.id); - if (projectUser) { - NcError.badRequest( - `${user.email} with role ${projectUser.roles} already exists in this project` - ); - } - - await ProjectUser.insert({ - project_id: req.params.projectId, - fk_user_id: user.id, - roles: req.body.roles || 'editor', - }); - - const cachedUser = await NocoCache.get( - `${CacheScope.USER}:${email}___${req.params.projectId}`, - CacheGetType.TYPE_OBJECT - ); - - if (cachedUser) { - cachedUser.roles = req.body.roles || 'editor'; - await NocoCache.set( - `${CacheScope.USER}:${email}___${req.params.projectId}`, - cachedUser - ); - } - - await Audit.insert({ - project_id: req.params.projectId, - op_type: 'AUTHENTICATION', - op_sub_type: 'INVITE', - user: req.user.email, - description: `invited ${email} to ${req.params.projectId} project `, - ip: req.clientIp, - }); - } else { - try { - // create new user with invite token - const { id } = await User.insert({ - invite_token, - invite_token_expires: new Date(Date.now() + 24 * 60 * 60 * 1000), - email, - roles: OrgUserRoles.VIEWER, - token_version: randomTokenString(), - }); - - // add user to project - await ProjectUser.insert({ - project_id: req.params.projectId, - fk_user_id: id, - roles: req.body.roles, - }); - - const count = await User.count(); - Tele.emit('evt', { evt_type: 'project:invite', count }); - - await Audit.insert({ - project_id: req.params.projectId, - op_type: 'AUTHENTICATION', - op_sub_type: 'INVITE', - user: req.user.email, - description: `invited ${email} to ${req.params.projectId} project `, - ip: req.clientIp, - }); - // in case of single user check for smtp failure - // and send back token if failed - if ( - emails.length === 1 && - !(await sendInviteEmail(email, invite_token, req)) - ) { - return res.json({ invite_token, email }); - } else { - sendInviteEmail(email, invite_token, req); - } - } catch (e) { - console.log(e); - if (emails.length === 1) { - return next(e); - } else { - error.push({ email, error: e.message }); - } - } - } - } - - if (emails.length === 1) { - res.json({ - msg: 'success', - }); - } else { - return res.json({ invite_token, emails, error }); - } +async function userInvite(req, res): Promise { + res.json( + await projectUserService.userInvite({ + projectId: req.params.projectId, + projectUser: req.body, + req, + }) + ); } // @ts-ignore async function projectUserUpdate(req, res, next): Promise { - if (!req?.body?.project_id) { - return next(new Error('Missing project id in request body.')); - } - - if ( - req.session?.passport?.user?.roles?.owner && - req.session?.passport?.user?.id === req.params.userId && - req.body.roles.indexOf('owner') === -1 - ) { - NcError.badRequest("Super admin can't remove Super role themselves"); - } - try { - const user = await User.get(req.params.userId); - - if (!user) { - NcError.badRequest(`User with id '${req.params.userId}' doesn't exist`); - } - - // todo: handle roles which contains super - if ( - !req.session?.passport?.user?.roles?.owner && - req.body.roles.indexOf('owner') > -1 - ) { - NcError.forbidden('Insufficient privilege to add super admin role.'); - } - - await ProjectUser.update( - req.params.projectId, - req.params.userId, - req.body.roles - ); - - await Audit.insert({ - op_type: 'AUTHENTICATION', - op_sub_type: 'ROLES_MANAGEMENT', - user: req.user.email, - description: `updated roles for ${user.email} with ${req.body.roles} `, - ip: req.clientIp, - }); - - res.json({ - msg: 'User details updated successfully', - }); - } catch (e) { - next(e); - } + res.json( + await projectUserService.projectUserUpdate({ + projectUser: req.body, + projectId: req.params.projectId, + userId: req.params.userId, + req, + }) + ); } async function projectUserDelete(req, res): Promise { - const project_id = req.params.projectId; - - if (req.session?.passport?.user?.id === req.params.userId) { - NcError.badRequest("Admin can't delete themselves!"); - } - - if (!req.session?.passport?.user?.roles?.owner) { - const user = await User.get(req.params.userId); - if (user.roles?.split(',').includes('super')) - NcError.forbidden('Insufficient privilege to delete a super admin user.'); - - const projectUser = await ProjectUser.get(project_id, req.params.userId); - if (projectUser?.roles?.split(',').includes('super')) - NcError.forbidden('Insufficient privilege to delete a owner user.'); - } - - await ProjectUser.delete(project_id, req.params.userId); + await projectUserService.projectUserDelete({ + projectId: req.params.projectId, + userId: req.params.userId, + req, + }); res.json({ msg: 'success', }); } async function projectUserInviteResend(req, res): Promise { - const user = await User.get(req.params.userId); - - if (!user) { - NcError.badRequest(`User with id '${req.params.userId}' not found`); - } - - req.body.roles = user.roles; - const invite_token = uuidv4(); - - await User.update(user.id, { - invite_token, - invite_token_expires: new Date(Date.now() + 24 * 60 * 60 * 1000), - }); - - const pluginData = await Noco.ncMeta.metaGet2(null, null, MetaTable.PLUGIN, { - category: PluginCategory.EMAIL, - active: true, - }); - - if (!pluginData) { - NcError.badRequest( - `No Email Plugin is found. Please go to App Store to configure first or copy the invitation URL to users instead.` - ); - } - - await sendInviteEmail(user.email, invite_token, req); - - await Audit.insert({ - op_type: 'AUTHENTICATION', - op_sub_type: 'RESEND_INVITE', - user: user.email, - description: `resent a invite to ${user.email} `, - ip: req.clientIp, - project_id: req.params.projectId, - }); - - res.json({ msg: 'success' }); -} - -export async function sendInviteEmail( - email: string, - token: string, - req: any -): Promise { - try { - const template = (await import('./userController/ui/emailTemplates/invite')) - .default; - - const emailAdapter = await NcPluginMgrv2.emailAdapter(); - - if (emailAdapter) { - await emailAdapter.mailSend({ - to: email, - subject: 'Verify email', - html: ejs.render(template, { - signupLink: `${req.ncSiteUrl}${ - Noco.getConfig()?.dashboardPath - }#/signup/${token}`, - projectName: req.body?.projectName, - roles: (req.body?.roles || '') - .split(',') - .map((r) => r.replace(/^./, (m) => m.toUpperCase())) - .join(', '), - adminEmail: req.session?.passport?.user?.email, - }), - }); - return true; - } - } catch (e) { - console.log( - 'Warning : `mailSend` failed, Please configure emailClient configuration.', - e.message - ); - throw e; - } + res.json( + await projectUserService.projectUserInviteResend({ + projectId: req.params.projectId, + userId: req.params.userId, + projectUser: req.body, + req, + }) + ); } const router = Router({ mergeParams: true }); @@ -311,13 +65,11 @@ router.get( router.post( '/api/v1/db/meta/projects/:projectId/users', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/ProjectUserReq'), ncMetaAclMw(userInvite, 'userInvite') ); router.patch( '/api/v1/db/meta/projects/:projectId/users/:userId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/ProjectUserReq'), ncMetaAclMw(projectUserUpdate, 'projectUserUpdate') ); router.delete( diff --git a/packages/nocodb/src/lib/controllers/sharedBaseController.ts b/packages/nocodb/src/lib/controllers/sharedBaseController.ts index a9bfcfbb48..1fd9fb27cd 100644 --- a/packages/nocodb/src/lib/controllers/sharedBaseController.ts +++ b/packages/nocodb/src/lib/controllers/sharedBaseController.ts @@ -1,93 +1,44 @@ import { Router } from 'express'; -import { Tele } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { v4 as uuidv4 } from 'uuid'; -import Project from '../models/Project'; -import { NcError } from '../meta/helpers/catchError'; -import { getAjvValidatorMw } from '../meta/api/helpers'; -// todo: load from config -const config = { - dashboardPath: '/nc', -}; +import { sharedBaseService } from '../services'; async function createSharedBaseLink(req, res): Promise { - const project = await Project.get(req.params.projectId); - - let roles = req.body?.roles; - if (!roles || (roles !== 'editor' && roles !== 'viewer')) { - roles = 'viewer'; - } - - if (!project) { - NcError.badRequest('Invalid project id'); - } - const data: any = { - uuid: uuidv4(), + const sharedBase = await sharedBaseService.createSharedBaseLink({ + projectId: req.params.projectId, + roles: req.body?.roles, password: req.body?.password, - roles, - }; - - await Project.update(project.id, data); + siteUrl: req.ncSiteUrl, + }); - data.url = `${req.ncSiteUrl}${config.dashboardPath}#/nc/base/${data.uuid}`; - delete data.password; - Tele.emit('evt', { evt_type: 'sharedBase:generated-link' }); - res.json(data); + res.json(sharedBase); } -async function updateSharedBaseLink(req, res): Promise { - const project = await Project.get(req.params.projectId); - - let roles = req.body?.roles; - if (!roles || (roles !== 'editor' && roles !== 'viewer')) { - roles = 'viewer'; - } - if (!project) { - NcError.badRequest('Invalid project id'); - } - const data: any = { - uuid: project.uuid || uuidv4(), +async function updateSharedBaseLink(req, res): Promise { + const sharedBase = await sharedBaseService.updateSharedBaseLink({ + projectId: req.params.projectId, + roles: req.body?.roles, password: req.body?.password, - roles, - }; - - await Project.update(project.id, data); + siteUrl: req.ncSiteUrl, + }); - data.url = `${req.ncSiteUrl}${config.dashboardPath}#/nc/base/${data.uuid}`; - delete data.password; - Tele.emit('evt', { evt_type: 'sharedBase:generated-link' }); - res.json(data); + res.json(sharedBase); } async function disableSharedBaseLink(req, res): Promise { - const project = await Project.get(req.params.projectId); + const sharedBase = await sharedBaseService.disableSharedBaseLink({ + projectId: req.params.projectId, + }); - if (!project) { - NcError.badRequest('Invalid project id'); - } - const data: any = { - uuid: null, - }; - - await Project.update(project.id, data); - Tele.emit('evt', { evt_type: 'sharedBase:disable-link' }); - res.json({ uuid: null }); + res.json(sharedBase); } async function getSharedBaseLink(req, res): Promise { - const project = await Project.get(req.params.projectId); - - if (!project) { - NcError.badRequest('Invalid project id'); - } - const data: any = { - uuid: project.uuid, - roles: project.roles, - }; - if (data.uuid) - data.url = `${req.ncSiteUrl}${config.dashboardPath}#/nc/base/${data.shared_base_id}`; + const sharedBase = await sharedBaseService.getSharedBaseLink({ + projectId: req.params.projectId, + siteUrl: req.ncSiteUrl, + }); - res.json(data); + res.json(sharedBase); } const router = Router({ mergeParams: true }); @@ -97,12 +48,10 @@ router.get( ); router.post( '/api/v1/db/meta/projects/:projectId/shared', - getAjvValidatorMw('swagger.json#/components/schemas/SharedBaseReq'), ncMetaAclMw(createSharedBaseLink, 'createSharedBaseLink') ); router.patch( '/api/v1/db/meta/projects/:projectId/shared', - getAjvValidatorMw('swagger.json#/components/schemas/SharedBaseReq'), ncMetaAclMw(updateSharedBaseLink, 'updateSharedBaseLink') ); router.delete( diff --git a/packages/nocodb/src/lib/controllers/sortController.ts b/packages/nocodb/src/lib/controllers/sortController.ts index 397b605f57..332e31e419 100644 --- a/packages/nocodb/src/lib/controllers/sortController.ts +++ b/packages/nocodb/src/lib/controllers/sortController.ts @@ -1,5 +1,4 @@ import { Request, Response, Router } from 'express'; -import { getAjvValidatorMw } from '../meta/api/helpers'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import { SortListType, SortReqType } from 'nocodb-sdk'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; @@ -59,7 +58,6 @@ router.get( router.post( '/api/v1/db/meta/views/:viewId/sorts/', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/SortReq'), ncMetaAclMw(sortCreate, 'sortCreate') ); @@ -72,7 +70,6 @@ router.get( router.patch( '/api/v1/db/meta/sorts/:sortId', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/SortReq'), ncMetaAclMw(sortUpdate, 'sortUpdate') ); router.delete( diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index d8987ef3cb..fdf6528111 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -1,11 +1,17 @@ import { Request, Response, Router } from 'express'; +import DOMPurify from 'isomorphic-dompurify'; import { TableListType, TableReqType, TableType } from 'nocodb-sdk'; -import { getAjvValidatorMw } from '../meta/api/helpers'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import { NcError } from '../meta/helpers/catchError'; +import getTableNameAlias from '../meta/helpers/getTableName'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; - +import Model from '../models/Model'; +import Project from '../models/Project'; +import { T } from 'nc-help'; import { tableService } from '../services'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; export async function tableList(req: Request, res: Response) { res.json( @@ -58,15 +64,108 @@ export async function tableReorder(req: Request, res: Response) { ); } +// todo: move to table service +export async function tableUpdate(req: Request, res) { + const model = await Model.get(req.params.tableId); + const project = await Project.getWithInfo( + req.body.project_id || (req as any).ncProjectId + ); + const base = project.bases.find((b) => b.id === model.base_id); -export async function tableUpdate(req: Request, res) { - tableService.tableUpdate({ - tableId: req.params.tableId, - table: req.body, - }) + if (model.project_id !== project.id) { + NcError.badRequest('Model does not belong to project'); + } + + // if meta present update meta and return + // todo: allow user to update meta and other prop in single api call + if ('meta' in req.body) { + await Model.updateMeta(req.params.tableId, req.body.meta); + + return res.json({ msg: 'success' }); + } + + if (!req.body.table_name) { + NcError.badRequest( + 'Missing table name `table_name` property in request body' + ); + } + + if (base.is_meta && project.prefix) { + if (!req.body.table_name.startsWith(project.prefix)) { + req.body.table_name = `${project.prefix}${req.body.table_name}`; + } + } + + req.body.table_name = DOMPurify.sanitize(req.body.table_name); + + // validate table name + if (/^\s+|\s+$/.test(req.body.table_name)) { + NcError.badRequest( + 'Leading or trailing whitespace not allowed in table names' + ); + } + + if ( + !(await Model.checkTitleAvailable({ + table_name: req.body.table_name, + project_id: project.id, + base_id: base.id, + })) + ) { + NcError.badRequest('Duplicate table name'); + } + + if (!req.body.title) { + req.body.title = getTableNameAlias( + req.body.table_name, + project.prefix, + base + ); + } + + if ( + !(await Model.checkAliasAvailable({ + title: req.body.title, + project_id: project.id, + base_id: base.id, + })) + ) { + NcError.badRequest('Duplicate table alias'); + } + + const sqlMgr = await ProjectMgrv2.getSqlMgr(project); + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + + let tableNameLengthLimit = 255; + const sqlClientType = sqlClient.knex.clientType(); + if (sqlClientType === 'mysql2' || sqlClientType === 'mysql') { + tableNameLengthLimit = 64; + } else if (sqlClientType === 'pg') { + tableNameLengthLimit = 63; + } else if (sqlClientType === 'mssql') { + tableNameLengthLimit = 128; + } + + if (req.body.table_name.length > tableNameLengthLimit) { + NcError.badRequest(`Table name exceeds ${tableNameLengthLimit} characters`); + } + + await Model.updateAliasAndTableName( + req.params.tableId, + req.body.title, + req.body.table_name + ); + + await sqlMgr.sqlOpPlus(base, 'tableRename', { + ...req.body, + tn: req.body.table_name, + tn_old: model.table_name, + }); + + T.emit('evt', { evt_type: 'table:updated' }); - res.json({ msg: 'success' }) + res.json({ msg: 'success' }); } const router = Router({ mergeParams: true }); @@ -83,13 +182,11 @@ router.get( router.post( '/api/v1/db/meta/projects/:projectId/tables', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/TableReq'), ncMetaAclMw(tableCreate, 'tableCreate') ); router.post( '/api/v1/db/meta/projects/:projectId/:baseId/tables', metaApiMetrics, - getAjvValidatorMw('swagger.json#/components/schemas/TableReq'), ncMetaAclMw(tableCreate, 'tableCreate') ); router.get( diff --git a/packages/nocodb/src/lib/controllers/userController/index.ts b/packages/nocodb/src/lib/controllers/userController/index.ts index 70d17b40ef..8843b2e5ea 100644 --- a/packages/nocodb/src/lib/controllers/userController/index.ts +++ b/packages/nocodb/src/lib/controllers/userController/index.ts @@ -340,11 +340,7 @@ const mapRoutes = (router) => { catchError(signin) ); router.get('/auth/user/me', extractProjectIdAndAuthenticate, catchError(me)); - router.post( - '/auth/password/forgot', - getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), - catchError(passwordForgot) - ); + router.post('/auth/password/forgot', catchError(passwordForgot)); router.post('/auth/token/validate/:tokenId', catchError(tokenValidate)); router.post( '/auth/password/reset/:tokenId', @@ -354,7 +350,6 @@ const mapRoutes = (router) => { router.post('/auth/email/validate/:tokenId', catchError(emailVerification)); router.post( '/user/password/change', - getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), ncMetaAclMw(passwordChange, 'passwordChange') ); router.post('/auth/token/refresh', catchError(refreshToken)); @@ -387,18 +382,13 @@ const mapRoutes = (router) => { extractProjectIdAndAuthenticate, catchError(me) ); - router.post( - '/api/v1/db/auth/password/forgot', - getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), - catchError(passwordForgot) - ); + router.post('/api/v1/db/auth/password/forgot', catchError(passwordForgot)); router.post( '/api/v1/db/auth/token/validate/:tokenId', catchError(tokenValidate) ); router.post( '/api/v1/db/auth/password/reset/:tokenId', - getAjvValidatorMw('swagger.json#/components/schemas/PasswordResetReq'), catchError(passwordReset) ); router.post( @@ -407,7 +397,6 @@ const mapRoutes = (router) => { ); router.post( '/api/v1/db/auth/password/change', - getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), ncMetaAclMw(passwordChange, 'passwordChange') ); router.post('/api/v1/db/auth/token/refresh', catchError(refreshToken)); @@ -432,11 +421,7 @@ const mapRoutes = (router) => { extractProjectIdAndAuthenticate, catchError(me) ); - router.post( - '/api/v1/auth/password/forgot', - getAjvValidatorMw('swagger.json#/components/schemas/ForgotPasswordReq'), - catchError(passwordForgot) - ); + router.post('/api/v1/auth/password/forgot', catchError(passwordForgot)); router.post( '/api/v1/auth/token/validate/:tokenId', catchError(tokenValidate) @@ -451,7 +436,6 @@ const mapRoutes = (router) => { ); router.post( '/api/v1/auth/password/change', - getAjvValidatorMw('swagger.json#/components/schemas/PasswordChangeReq'), ncMetaAclMw(passwordChange, 'passwordChange') ); router.post('/api/v1/auth/token/refresh', catchError(refreshToken)); diff --git a/packages/nocodb/src/lib/meta/api/helpers/apiHelpers.ts b/packages/nocodb/src/lib/meta/api/helpers/apiHelpers.ts index 25a077a52d..e1633ccc89 100644 --- a/packages/nocodb/src/lib/meta/api/helpers/apiHelpers.ts +++ b/packages/nocodb/src/lib/meta/api/helpers/apiHelpers.ts @@ -2,7 +2,7 @@ import { NextFunction, Request, Response } from 'express'; import Ajv, { ErrorObject } from 'ajv'; // @ts-ignore import swagger from '../../../../schema/swagger.json'; -import { NcError } from '../../helpers/catchError' +import { NcError } from '../../helpers/catchError'; export function parseHrtimeToSeconds(hrtime) { const seconds = (hrtime[0] + hrtime[1] / 1e6).toFixed(3); @@ -38,7 +38,7 @@ export const getAjvValidatorMw = (schema) => { }; // a function to validate the payload against the schema -export const ajvValidator = (schema, payload) => { +export const validatePayload = (schema, payload) => { // Validate the request body against the schema const valid = ajv.validate( typeof schema === 'string' ? { $ref: schema } : schema, @@ -55,4 +55,4 @@ export const ajvValidator = (schema, payload) => { errors, }); } -} +}; diff --git a/packages/nocodb/src/lib/models/Hook.ts b/packages/nocodb/src/lib/models/Hook.ts index 1761dc8c12..ea0db07985 100644 --- a/packages/nocodb/src/lib/models/Hook.ts +++ b/packages/nocodb/src/lib/models/Hook.ts @@ -1,42 +1,42 @@ -import { BoolType, HookReqType, HookType } from 'nocodb-sdk' +import { BoolType, HookReqType, HookType } from 'nocodb-sdk'; import { CacheDelDirection, CacheGetType, CacheScope, MetaTable, -} from '../utils/globals' -import Noco from '../Noco' -import Model from './Model' -import NocoCache from '../cache/NocoCache' -import Filter from './Filter' -import HookFilter from './HookFilter' -import { extractProps } from '../meta/helpers/extractProps' +} from '../utils/globals'; +import Noco from '../Noco'; +import Model from './Model'; +import NocoCache from '../cache/NocoCache'; +import Filter from './Filter'; +import HookFilter from './HookFilter'; +import { extractProps } from '../meta/helpers/extractProps'; export default class Hook implements HookType { - id?: string - fk_model_id?: string - title?: string - description?: string - env?: string - type?: string - event?: 'after' | 'before' - operation?: 'insert' | 'delete' | 'update' - async?: BoolType - payload?: string - url?: string - headers?: string - condition?: BoolType - notification?: string - retries?: number - retry_interval?: number - timeout?: number - active?: BoolType + id?: string; + fk_model_id?: string; + title?: string; + description?: string; + env?: string; + type?: string; + event?: 'after' | 'before'; + operation?: 'insert' | 'delete' | 'update'; + async?: BoolType; + payload?: string; + url?: string; + headers?: string; + condition?: BoolType; + notification?: string; + retries?: number; + retry_interval?: number; + timeout?: number; + active?: BoolType; - project_id?: string - base_id?: string + project_id?: string; + base_id?: string; constructor(hook: Partial) { - Object.assign(this, hook) + Object.assign(this, hook); } public static async get(hookId: string, ncMeta = Noco.ncMeta) { @@ -44,17 +44,17 @@ export default class Hook implements HookType { hookId && (await NocoCache.get( `${CacheScope.HOOK}:${hookId}`, - CacheGetType.TYPE_OBJECT, - )) + CacheGetType.TYPE_OBJECT + )); if (!hook) { - hook = await ncMeta.metaGet2(null, null, MetaTable.HOOKS, hookId) - await NocoCache.set(`${CacheScope.HOOK}:${hookId}`, hook) + hook = await ncMeta.metaGet2(null, null, MetaTable.HOOKS, hookId); + await NocoCache.set(`${CacheScope.HOOK}:${hookId}`, hook); } - return hook && new Hook(hook) + return hook && new Hook(hook); } public async getFilters(ncMeta = Noco.ncMeta) { - return await Filter.rootFilterListByHook({ hookId: this.id }, ncMeta) + return await Filter.rootFilterListByHook({ hookId: this.id }, ncMeta); } // public static async insert(hook: Partial) { @@ -81,9 +81,9 @@ export default class Hook implements HookType { event?: 'after' | 'before'; operation?: 'insert' | 'delete' | 'update'; }, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ) { - let hooks = await NocoCache.getList(CacheScope.HOOK, [param.fk_model_id]) + let hooks = await NocoCache.getList(CacheScope.HOOK, [param.fk_model_id]); if (!hooks.length) { hooks = await ncMeta.metaList(null, null, MetaTable.HOOKS, { condition: { @@ -96,29 +96,31 @@ export default class Hook implements HookType { orderBy: { created_at: 'asc', }, - }) - await NocoCache.setList(CacheScope.HOOK, [param.fk_model_id], hooks) + }); + await NocoCache.setList(CacheScope.HOOK, [param.fk_model_id], hooks); } // filter event & operation if (param.event) { hooks = hooks.filter( - (h) => h.event?.toLowerCase() === param.event?.toLowerCase(), - ) + (h) => h.event?.toLowerCase() === param.event?.toLowerCase() + ); } if (param.operation) { hooks = hooks.filter( - (h) => h.operation?.toLowerCase() === param.operation?.toLowerCase(), - ) + (h) => h.operation?.toLowerCase() === param.operation?.toLowerCase() + ); } - return hooks?.map((h) => new Hook(h)) + return hooks?.map((h) => new Hook(h)); } public static async insert( - hook: Partial, - ncMeta = Noco.ncMeta, + hook: Partial< + Hook & { + created_at?; + updated_at?; + } + >, + ncMeta = Noco.ncMeta ) { const insertObj = extractProps(hook, [ 'fk_model_id', @@ -140,49 +142,49 @@ export default class Hook implements HookType { 'base_id', 'created_at', 'updated_at', - ]) + ]); if (insertObj.event) { - insertObj.event = insertObj.event.toLowerCase() as 'after' | 'before' + insertObj.event = insertObj.event.toLowerCase() as 'after' | 'before'; } if (insertObj.operation) { insertObj.operation = insertObj.operation.toLowerCase() as | 'insert' | 'delete' - | 'update' + | 'update'; } if (insertObj.notification && typeof insertObj.notification === 'object') { - insertObj.notification = JSON.stringify(insertObj.notification) + insertObj.notification = JSON.stringify(insertObj.notification); } if (!(hook.project_id && hook.base_id)) { - const model = await Model.getByIdOrName({ id: hook.fk_model_id }, ncMeta) - insertObj.project_id = model.project_id - insertObj.base_id = model.base_id + const model = await Model.getByIdOrName({ id: hook.fk_model_id }, ncMeta); + insertObj.project_id = model.project_id; + insertObj.base_id = model.base_id; } const { id } = await ncMeta.metaInsert2( null, null, MetaTable.HOOKS, - insertObj, - ) + insertObj + ); await NocoCache.appendToList( CacheScope.HOOK, [hook.fk_model_id], - `${CacheScope.HOOK}:${id}`, - ) + `${CacheScope.HOOK}:${id}` + ); - return this.get(id, ncMeta) + return this.get(id, ncMeta); } public static async update( hookId: string, hook: Partial, - ncMeta = Noco.ncMeta, + ncMeta = Noco.ncMeta ) { const updateObj = extractProps(hook, [ 'title', @@ -201,38 +203,38 @@ export default class Hook implements HookType { 'retry_interval', 'timeout', 'active', - ]) + ]); if (updateObj.event) { - updateObj.event = updateObj.event.toLowerCase() as 'after' | 'before' + updateObj.event = updateObj.event.toLowerCase() as 'after' | 'before'; } if (updateObj.operation) { updateObj.operation = updateObj.operation.toLowerCase() as | 'insert' | 'delete' - | 'update' + | 'update'; } if (updateObj.notification && typeof updateObj.notification === 'object') { - updateObj.notification = JSON.stringify(updateObj.notification) + updateObj.notification = JSON.stringify(updateObj.notification); } // get existing cache - const key = `${CacheScope.HOOK}:${hookId}` - let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) + const key = `${CacheScope.HOOK}:${hookId}`; + let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); if (o) { // update data - o = { ...o, ...updateObj } + o = { ...o, ...updateObj }; // replace notification - o.notification = updateObj.notification + o.notification = updateObj.notification; // set cache - await NocoCache.set(key, o) + await NocoCache.set(key, o); } // set meta - await ncMeta.metaUpdate(null, null, MetaTable.HOOKS, updateObj, hookId) + await ncMeta.metaUpdate(null, null, MetaTable.HOOKS, updateObj, hookId); - return this.get(hookId, ncMeta) + return this.get(hookId, ncMeta); } static async delete(hookId: any, ncMeta = Noco.ncMeta) { @@ -243,22 +245,22 @@ export default class Hook implements HookType { MetaTable.FILTER_EXP, { condition: { fk_hook_id: hookId }, - }, - ) + } + ); for (const filter of filterList) { await NocoCache.deepDel( CacheScope.FILTER_EXP, `${CacheScope.FILTER_EXP}:${filter.id}`, - CacheDelDirection.CHILD_TO_PARENT, - ) - await HookFilter.delete(filter.id) + CacheDelDirection.CHILD_TO_PARENT + ); + await HookFilter.delete(filter.id); } // Delete Hook await NocoCache.deepDel( CacheScope.HOOK, `${CacheScope.HOOK}:${hookId}`, - CacheDelDirection.CHILD_TO_PARENT, - ) - return await ncMeta.metaDelete(null, null, MetaTable.HOOKS, hookId) + CacheDelDirection.CHILD_TO_PARENT + ); + return await ncMeta.metaDelete(null, null, MetaTable.HOOKS, hookId); } } diff --git a/packages/nocodb/src/lib/services/apiTokenService.ts b/packages/nocodb/src/lib/services/apiTokenService.ts index af94b7936b..329bcb305c 100644 --- a/packages/nocodb/src/lib/services/apiTokenService.ts +++ b/packages/nocodb/src/lib/services/apiTokenService.ts @@ -1,5 +1,6 @@ import { ApiTokenReqType, OrgUserRoles } from 'nocodb-sdk'; import { T } from 'nc-help'; +import { validatePayload } from '../meta/api/helpers'; import { NcError } from '../meta/helpers/catchError'; import ApiToken from '../models/ApiToken'; import User from '../models/User'; @@ -11,6 +12,11 @@ export async function apiTokenCreate(param: { userId: string; tokenBody: ApiTokenReqType; }) { + await validatePayload( + 'swagger.json#/components/schemas/ApiTokenReq', + param.tokenBody + ); + T.emit('evt', { evt_type: 'apiToken:created' }); return ApiToken.insert({ ...param.tokenBody, fk_user_id: param.userId }); } diff --git a/packages/nocodb/src/lib/services/auditService.ts b/packages/nocodb/src/lib/services/auditService.ts index 211c8d4f2d..f5f2724ba3 100644 --- a/packages/nocodb/src/lib/services/auditService.ts +++ b/packages/nocodb/src/lib/services/auditService.ts @@ -2,6 +2,7 @@ import { AuditRowUpdatePayloadType, CommentRowPayloadType, } from 'nocodb-sdk/build/main/lib/CustomAPI'; +import { validatePayload } from '../meta/api/helpers'; import Audit from '../models/Audit'; import { AuditOperationSubTypes, AuditOperationTypes } from 'nocodb-sdk'; import Model from '../models/Model'; @@ -14,6 +15,11 @@ export async function commentRow(param: { body: CommentRowPayloadType; user: any; }) { + await validatePayload( + 'swagger.json#/components/schemas/CommentReq', + param.body + ); + return await Audit.insert({ ...param.body, user: param.user?.email, @@ -25,6 +31,11 @@ export async function auditRowUpdate(param: { rowId: string; body: AuditRowUpdatePayloadType; }) { + await validatePayload( + 'swagger.json#/components/schemas/AuditRowUpdateReq', + param.body + ); + const model = await Model.getByIdOrName({ id: param.body.fk_model_id }); return await Audit.insert({ fk_model_id: param.body.fk_model_id, diff --git a/packages/nocodb/src/lib/services/baseService.ts b/packages/nocodb/src/lib/services/baseService.ts index dbb5001bf0..21adb99ac8 100644 --- a/packages/nocodb/src/lib/services/baseService.ts +++ b/packages/nocodb/src/lib/services/baseService.ts @@ -3,7 +3,7 @@ import { BaseReqType } from 'nocodb-sdk'; import { syncBaseMigration } from '../meta/helpers/syncMigration'; import Base from '../models/Base'; import { T } from 'nc-help'; -import { populateMeta } from '../meta/api/helpers'; +import { populateMeta, validatePayload } from '../meta/api/helpers'; export async function baseGetWithConfig(param: { baseId: any }) { const base = await Base.get(param.baseId); @@ -18,6 +18,8 @@ export async function baseUpdate(param: { base: BaseReqType; projectId: string; }) { + validatePayload('swagger.json#/components/schemas/BaseReq', param.base); + const baseBody = param.base; const project = await Project.getWithInfo(param.projectId); const base = await Base.updateBase(param.baseId, { @@ -53,6 +55,8 @@ export async function baseCreate(param: { projectId: string; base: BaseReqType; }) { + validatePayload('swagger.json#/components/schemas/BaseReq', param.base); + // type | base | projectId const baseBody = param.base; const project = await Project.getWithInfo(param.projectId); diff --git a/packages/nocodb/src/lib/services/columnService.ts b/packages/nocodb/src/lib/services/columnService.ts index f2c741f546..185186d0a2 100644 --- a/packages/nocodb/src/lib/services/columnService.ts +++ b/packages/nocodb/src/lib/services/columnService.ts @@ -18,6 +18,7 @@ import { generateFkName, randomID, validateLookupPayload, + validatePayload, validateRequiredField, validateRollupPayload, } from '../meta/api/helpers'; @@ -835,6 +836,8 @@ export async function columnAdd(param: { tableId: string; column: ColumnReqType; }) { + validatePayload('swagger.json#/components/schemas/ColumnReq', param.column); + const table = await Model.getWithInfo({ id: param.tableId, }); diff --git a/packages/nocodb/src/lib/services/dataService/index.ts b/packages/nocodb/src/lib/services/dataService/index.ts index 1f1d09b20a..c125530352 100644 --- a/packages/nocodb/src/lib/services/dataService/index.ts +++ b/packages/nocodb/src/lib/services/dataService/index.ts @@ -795,4 +795,4 @@ export async function relationDataAdd(param: { return true; } -export * from './helpers' +export * from './helpers'; diff --git a/packages/nocodb/src/lib/services/filterService.ts b/packages/nocodb/src/lib/services/filterService.ts index c52089118b..351d312378 100644 --- a/packages/nocodb/src/lib/services/filterService.ts +++ b/packages/nocodb/src/lib/services/filterService.ts @@ -1,4 +1,5 @@ import { FilterReqType } from 'nocodb-sdk'; +import { validatePayload } from '../meta/api/helpers'; import Filter from '../models/Filter'; import { T } from 'nc-help'; @@ -6,6 +7,8 @@ export async function hookFilterCreate(param: { filter: FilterReqType; hookId: any; }) { + validatePayload('swagger.json#/components/schemas/FilterReq', param.filter); + const filter = await Filter.insert({ ...param.filter, fk_hook_id: param.hookId, @@ -29,6 +32,8 @@ export async function filterCreate(param: { filter: FilterReqType; viewId: string; }) { + validatePayload('swagger.json#/components/schemas/FilterReq', param.filter); + const filter = await Filter.insert({ ...param.filter, fk_view_id: param.viewId, @@ -42,6 +47,8 @@ export async function filterUpdate(param: { filter: FilterReqType; filterId: string; }) { + validatePayload('swagger.json#/components/schemas/FilterReq', param.filter); + // todo: type correction const filter = await Filter.update(param.filterId, param.filter as Filter); diff --git a/packages/nocodb/src/lib/services/formViewColumnService.ts b/packages/nocodb/src/lib/services/formViewColumnService.ts index b91d171f3b..ecceec1f81 100644 --- a/packages/nocodb/src/lib/services/formViewColumnService.ts +++ b/packages/nocodb/src/lib/services/formViewColumnService.ts @@ -1,3 +1,4 @@ +import { validatePayload } from '../meta/api/helpers'; import { FormViewColumn } from '../models'; import { T } from 'nc-help'; export async function columnUpdate(param: { @@ -5,6 +6,11 @@ export async function columnUpdate(param: { // todo: replace with FormColumnReq formViewColumn: FormViewColumn; }) { + validatePayload( + 'swagger.json#/components/schemas/FormColumnReq', + param.formViewColumn + ); + T.emit('evt', { evt_type: 'formViewColumn:updated' }); return await FormViewColumn.update( param.formViewColumnId, diff --git a/packages/nocodb/src/lib/services/formViewService.ts b/packages/nocodb/src/lib/services/formViewService.ts index eec4c66624..94c6f5c9a5 100644 --- a/packages/nocodb/src/lib/services/formViewService.ts +++ b/packages/nocodb/src/lib/services/formViewService.ts @@ -1,5 +1,6 @@ import { T } from 'nc-help'; import { FormReqType, ViewTypes } from 'nocodb-sdk'; +import { validatePayload } from '../meta/api/helpers'; import { FormView, View } from '../models'; export async function formViewGet(param: { formViewId: string }) { @@ -11,6 +12,8 @@ export async function formViewCreate(param: { tableId: string; body: FormReqType; }) { + validatePayload('swagger.json#/components/schemas/FormCreateReq', param.body); + T.emit('evt', { evt_type: 'vtable:created', show_as: 'form' }); const view = await View.insert({ ...param.body, @@ -26,6 +29,8 @@ export async function formViewUpdate(param: { formViewId: string; body: FormReqType; }) { + validatePayload('swagger.json#/components/schemas/FormReq', param.body); + T.emit('evt', { evt_type: 'view:updated', type: 'grid' }); await FormView.update(param.formViewId, param.body); } diff --git a/packages/nocodb/src/lib/services/galleryViewService.ts b/packages/nocodb/src/lib/services/galleryViewService.ts index 9e8ab05d5d..8c564fa74d 100644 --- a/packages/nocodb/src/lib/services/galleryViewService.ts +++ b/packages/nocodb/src/lib/services/galleryViewService.ts @@ -1,5 +1,6 @@ import { GalleryReqType, ViewTypes } from 'nocodb-sdk'; import { T } from 'nc-help'; +import { validatePayload } from '../meta/api/helpers'; import { GalleryView, View } from '../models'; export async function galleryViewGet(param: { galleryViewId: string }) { @@ -10,6 +11,8 @@ export async function galleryViewCreate(param: { tableId: string; gallery: GalleryReqType; }) { + validatePayload('swagger.json#/components/schemas/GalleryReq', param.gallery); + T.emit('evt', { evt_type: 'vtable:created', show_as: 'gallery' }); const view = await View.insert({ ...param.gallery, @@ -24,6 +27,8 @@ export async function galleryViewUpdate(param: { galleryViewId: string; gallery: GalleryReqType; }) { + validatePayload('swagger.json#/components/schemas/GalleryReq', param.gallery); + T.emit('evt', { evt_type: 'view:updated', type: 'gallery' }); await GalleryView.update(param.galleryViewId, param.gallery); } diff --git a/packages/nocodb/src/lib/services/gridViewColumnService.ts b/packages/nocodb/src/lib/services/gridViewColumnService.ts index e24dd07177..70d5d3df76 100644 --- a/packages/nocodb/src/lib/services/gridViewColumnService.ts +++ b/packages/nocodb/src/lib/services/gridViewColumnService.ts @@ -1,4 +1,5 @@ import { GridColumnReqType } from 'nocodb-sdk'; +import { validatePayload } from '../meta/api/helpers'; import GridViewColumn from '../models/GridViewColumn'; import { T } from 'nc-help'; @@ -10,6 +11,8 @@ export async function gridColumnUpdate(param: { gridViewColumnId: string; grid: GridColumnReqType; }) { + validatePayload('swagger.json#/components/schemas/GridColumnReq', param.grid); + T.emit('evt', { evt_type: 'gridViewColumn:updated' }); return await GridViewColumn.update(param.gridViewColumnId, param.grid); } diff --git a/packages/nocodb/src/lib/services/gridViewService.ts b/packages/nocodb/src/lib/services/gridViewService.ts index c99f63552e..92d01ced57 100644 --- a/packages/nocodb/src/lib/services/gridViewService.ts +++ b/packages/nocodb/src/lib/services/gridViewService.ts @@ -1,5 +1,6 @@ import { T } from 'nc-help'; import { GridReqType, ViewTypes } from 'nocodb-sdk'; +import { validatePayload } from '../meta/api/helpers'; import { View } from '../models'; import { GridView } from '../models'; @@ -7,6 +8,8 @@ export async function gridViewCreate(param: { tableId: string; grid: GridReqType; }) { + validatePayload('swagger.json#/components/schemas/GridReq', param.grid); + const view = await View.insert({ ...param.grid, // todo: sanitize @@ -17,6 +20,7 @@ export async function gridViewCreate(param: { return view; } +// todo: json schema validation export async function gridViewUpdate(param: { viewId: string; grid: GridReqType; diff --git a/packages/nocodb/src/lib/services/hookFilterService.ts b/packages/nocodb/src/lib/services/hookFilterService.ts index d4c0b2636d..b8b1e04a42 100644 --- a/packages/nocodb/src/lib/services/hookFilterService.ts +++ b/packages/nocodb/src/lib/services/hookFilterService.ts @@ -1,5 +1,6 @@ import { T } from 'nc-help'; import { FilterReqType } from 'nocodb-sdk'; +import { validatePayload } from '../meta/api/helpers'; import Filter from '../models/Filter'; export async function filterGet(param: { hookId: string }) { @@ -32,6 +33,8 @@ export async function filterCreate(param: { hookId: string; filter: FilterReqType; }) { + validatePayload('swagger.json#/components/schemas/FilterReq', param.filter); + const filter = await Filter.insert({ ...param.filter, fk_hook_id: param.hookId, @@ -46,6 +49,8 @@ export async function filterUpdate(param: { filterId: string; filter: FilterReqType; }) { + validatePayload('swagger.json#/components/schemas/FilterReq', param.filter); + const filter = await Filter.update(param.filterId, { ...param.filter, fk_hook_id: param.hookId, diff --git a/packages/nocodb/src/lib/services/hookService.ts b/packages/nocodb/src/lib/services/hookService.ts index 2e20732c17..9acf6d4b48 100644 --- a/packages/nocodb/src/lib/services/hookService.ts +++ b/packages/nocodb/src/lib/services/hookService.ts @@ -1,4 +1,5 @@ import { T } from 'nc-help'; +import { validatePayload } from '../meta/api/helpers'; import { Hook, Model } from '../models'; import { HookReqType, HookTestReqType } from 'nocodb-sdk'; @@ -14,6 +15,8 @@ export async function hookCreate(param: { tableId: string; hook: HookReqType; }) { + validatePayload('swagger.json#/components/schemas/HookReq', param.hook); + T.emit('evt', { evt_type: 'webhooks:created' }); // todo: type correction const hook = await Hook.insert({ @@ -30,6 +33,8 @@ export async function hookDelete(param: { hookId: string }) { } export async function hookUpdate(param: { hookId: string; hook: HookReqType }) { + validatePayload('swagger.json#/components/schemas/HookReq', param.hook); + T.emit('evt', { evt_type: 'webhooks:updated' }); // todo: correction in swagger @@ -40,6 +45,11 @@ export async function hookTest(param: { tableId: string; hookTest: HookTestReqType; }) { + validatePayload( + 'swagger.json#/components/schemas/HookTestReq', + param.hookTest + ); + const model = await Model.getByIdOrName({ id: param.tableId }); const { diff --git a/packages/nocodb/src/lib/services/kanbanViewService.ts b/packages/nocodb/src/lib/services/kanbanViewService.ts index 160044e26a..e6561ae323 100644 --- a/packages/nocodb/src/lib/services/kanbanViewService.ts +++ b/packages/nocodb/src/lib/services/kanbanViewService.ts @@ -1,4 +1,5 @@ import { KanbanReqType, ViewTypes } from 'nocodb-sdk'; +import { validatePayload } from '../meta/api/helpers'; import { KanbanView, View } from '../models'; import { T } from 'nc-help'; @@ -10,7 +11,8 @@ export async function kanbanViewCreate(param: { tableId: string; kanban: KanbanReqType; }) { - T.emit('evt', { evt_type: 'vtable:created', show_as: 'kanban' }); + validatePayload('swagger.json#/components/schemas/KanbanReq', param.kanban), + T.emit('evt', { evt_type: 'vtable:created', show_as: 'kanban' }); const view = await View.insert({ ...param.kanban, // todo: sanitize @@ -24,6 +26,10 @@ export async function kanbanViewUpdate(param: { kanbanViewId: string; kanban: KanbanReqType; }) { - T.emit('evt', { evt_type: 'view:updated', type: 'kanban' }); + validatePayload( + 'swagger.json#/components/schemas/KanbanUpdateReq', + param.kanban + ), + T.emit('evt', { evt_type: 'view:updated', type: 'kanban' }); return await KanbanView.update(param.kanbanViewId, param.kanban); } diff --git a/packages/nocodb/src/lib/services/modelVisibilityService.ts b/packages/nocodb/src/lib/services/modelVisibilityService.ts index eb2f21ec7e..719d3de487 100644 --- a/packages/nocodb/src/lib/services/modelVisibilityService.ts +++ b/packages/nocodb/src/lib/services/modelVisibilityService.ts @@ -1,4 +1,5 @@ import { VisibilityRuleReqType } from 'nocodb-sdk'; +import { validatePayload } from '../meta/api/helpers'; import { NcError } from '../meta/helpers/catchError'; import ModelRoleVisibility from '../models/ModelRoleVisibility'; import { T } from 'nc-help'; @@ -8,7 +9,11 @@ export async function xcVisibilityMetaSetAll(param: { visibilityRule: VisibilityRuleReqType; projectId: string; }) { - T.emit('evt', { evt_type: 'uiAcl:updated' }); + validatePayload( + 'swagger.json#/components/schemas/VisibilityRuleReq', + param.visibilityRule + ), + T.emit('evt', { evt_type: 'uiAcl:updated' }); for (const d of param.visibilityRule) { for (const role of Object.keys(d.disabled)) { const view = await View.get(d.id); diff --git a/packages/nocodb/src/lib/services/orgLicenseService.ts b/packages/nocodb/src/lib/services/orgLicenseService.ts index c39d7814a9..d6f8033575 100644 --- a/packages/nocodb/src/lib/services/orgLicenseService.ts +++ b/packages/nocodb/src/lib/services/orgLicenseService.ts @@ -1,4 +1,5 @@ import { NC_LICENSE_KEY } from '../constants'; +import { validatePayload } from '../meta/api/helpers'; import Store from '../models/Store'; import Noco from '../Noco'; @@ -9,6 +10,8 @@ export async function licenseGet() { } export async function licenseSet(param: { key: string }) { + validatePayload('swagger.json#/components/schemas/LicenseReq', param); + await Store.saveOrUpdate({ value: param.key, key: NC_LICENSE_KEY }); await Noco.loadEEState(); return true; diff --git a/packages/nocodb/src/lib/services/orgTokenService.ts b/packages/nocodb/src/lib/services/orgTokenService.ts index 30228c336a..fdee8e5fbe 100644 --- a/packages/nocodb/src/lib/services/orgTokenService.ts +++ b/packages/nocodb/src/lib/services/orgTokenService.ts @@ -1,4 +1,5 @@ import { ApiTokenReqType, OrgUserRoles } from 'nocodb-sdk'; +import { validatePayload } from '../meta/api/helpers'; import { User } from '../models'; import ApiToken from '../models/ApiToken'; import { T } from 'nc-help'; @@ -32,6 +33,11 @@ export async function apiTokenCreate(param: { user: User; apiToken: ApiTokenReqType; }) { + validatePayload( + 'swagger.json#/components/schemas/ApiTokenReq', + param.apiToken + ); + T.emit('evt', { evt_type: 'org:apiToken:created' }); return await ApiToken.insert({ ...param.apiToken, diff --git a/packages/nocodb/src/lib/services/orgUserService.ts b/packages/nocodb/src/lib/services/orgUserService.ts index 318afea8c0..b4e2d2adac 100644 --- a/packages/nocodb/src/lib/services/orgUserService.ts +++ b/packages/nocodb/src/lib/services/orgUserService.ts @@ -8,6 +8,7 @@ import { v4 as uuidv4 } from 'uuid'; import validator from 'validator'; import { OrgUserRoles } from 'nocodb-sdk'; import { NC_APP_SETTINGS } from '../constants'; +import { validatePayload } from '../meta/api/helpers'; import { Audit, ProjectUser, Store, SyncSource, User } from '../models'; import Noco from '../Noco'; import { MetaTable } from '../utils/globals'; @@ -35,6 +36,8 @@ export async function userUpdate(param: { user: Partial; userId: string; }) { + validatePayload('swagger.json#/components/schemas/OrgUserReq', param.user); + const updateBody = extractProps(param.user, ['roles']); const user = await User.get(param.userId); @@ -95,6 +98,8 @@ export async function userAdd(param: { // todo: refactor req: any; }) { + validatePayload('swagger.json#/components/schemas/OrgUserReq', param.user); + // allow only viewer or creator role if ( param.user.roles && diff --git a/packages/nocodb/src/lib/services/pluginService.ts b/packages/nocodb/src/lib/services/pluginService.ts index 2cb885e001..67302d8ce7 100644 --- a/packages/nocodb/src/lib/services/pluginService.ts +++ b/packages/nocodb/src/lib/services/pluginService.ts @@ -1,4 +1,5 @@ import { T } from 'nc-help'; +import { validatePayload } from '../meta/api/helpers'; import { Plugin } from '../models'; import { PluginTestReqType, PluginType } from 'nocodb-sdk'; import NcPluginMgrv2 from '../meta/helpers/NcPluginMgrv2'; @@ -8,6 +9,8 @@ export async function pluginList() { } export async function pluginTest(param: { body: PluginTestReqType }) { + validatePayload('swagger.json#/components/schemas/PluginTestReq', param.body); + T.emit('evt', { evt_type: 'plugin:tested' }); return await NcPluginMgrv2.test(param.body); } @@ -19,6 +22,8 @@ export async function pluginUpdate(param: { pluginId: string; plugin: PluginType; }) { + validatePayload('swagger.json#/components/schemas/PluginReq', param.plugin); + const plugin = await Plugin.update(param.pluginId, param.plugin); T.emit('evt', { evt_type: plugin.active ? 'plugin:installed' : 'plugin:uninstalled', diff --git a/packages/nocodb/src/lib/services/projectService.ts b/packages/nocodb/src/lib/services/projectService.ts index eaf4b98a64..08a848f7a5 100644 --- a/packages/nocodb/src/lib/services/projectService.ts +++ b/packages/nocodb/src/lib/services/projectService.ts @@ -1,7 +1,7 @@ import DOMPurify from 'isomorphic-dompurify'; import { OrgUserRoles, ProjectReqType } from 'nocodb-sdk'; import { promisify } from 'util'; -import { populateMeta } from '../meta/api/helpers'; +import { populateMeta, validatePayload } from '../meta/api/helpers'; import { extractPropsAndSanitize } from '../meta/helpers/extractProps'; import syncMigration from '../meta/helpers/syncMigration'; import Project from '../models/Project'; @@ -15,6 +15,8 @@ export async function projectCreate(param: { project: ProjectReqType; user: any; }) { + validatePayload('swagger.json#/components/schemas/ProjectReq', param.project); + const projectBody: ProjectReqType & Record = param.project; if (!projectBody.external) { const ranId = nanoid(); diff --git a/packages/nocodb/src/lib/services/projectUserService.ts b/packages/nocodb/src/lib/services/projectUserService.ts index 2ff67913d4..12e6a92735 100644 --- a/packages/nocodb/src/lib/services/projectUserService.ts +++ b/packages/nocodb/src/lib/services/projectUserService.ts @@ -1,5 +1,6 @@ import { OrgUserRoles, ProjectUserReqType } from 'nocodb-sdk'; import { T } from 'nc-help'; +import { validatePayload } from '../meta/api/helpers'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; import ProjectUser from '../models/ProjectUser'; import validator from 'validator'; @@ -33,6 +34,11 @@ export async function userInvite(param: { projectUser: ProjectUserReqType; req: any; }): Promise { + validatePayload( + 'swagger.json#/components/schemas/ProjectUserReq', + param.projectUser + ); + const emails = (param.projectUser.email || '') .toLowerCase() .split(/\s*,\s*/) @@ -157,6 +163,11 @@ export async function projectUserUpdate(param: { req: any; projectId: string; }): Promise { + validatePayload( + 'swagger.json#/components/schemas/ProjectUserReq', + param.projectUser + ); + // todo: use param.projectId if (!param.projectUser?.project_id) { NcError.badRequest('Missing project id in request body.'); diff --git a/packages/nocodb/src/lib/services/sharedBaseService.ts b/packages/nocodb/src/lib/services/sharedBaseService.ts index 53213d7627..a3549781f4 100644 --- a/packages/nocodb/src/lib/services/sharedBaseService.ts +++ b/packages/nocodb/src/lib/services/sharedBaseService.ts @@ -1,5 +1,6 @@ import { T } from 'nc-help'; import { v4 as uuidv4 } from 'uuid'; +import { validatePayload } from '../meta/api/helpers'; import Project from '../models/Project'; import { NcError } from '../meta/helpers/catchError'; // todo: load from config @@ -13,6 +14,8 @@ export async function createSharedBaseLink(param: { password: string; siteUrl: string; }): Promise { + validatePayload('swagger.json#/components/schemas/SharedBaseReq', param); + const project = await Project.get(param.projectId); let roles = param?.roles; @@ -44,6 +47,8 @@ export async function updateSharedBaseLink(param: { password: string; siteUrl: string; }): Promise { + validatePayload('swagger.json#/components/schemas/SharedBaseReq', param); + const project = await Project.get(param.projectId); let roles = param.roles; diff --git a/packages/nocodb/src/lib/services/sortService.ts b/packages/nocodb/src/lib/services/sortService.ts index e3931bad2c..c2d05948c1 100644 --- a/packages/nocodb/src/lib/services/sortService.ts +++ b/packages/nocodb/src/lib/services/sortService.ts @@ -1,4 +1,5 @@ import { SortReqType } from 'nocodb-sdk'; +import { validatePayload } from '../meta/api/helpers'; import Sort from '../models/Sort'; import { T } from 'nc-help'; @@ -13,12 +14,16 @@ export async function sortDelete(param: { sortId: string }) { } export async function sortUpdate(param: { sortId: any; sort: SortReqType }) { + validatePayload('swagger.json#/components/schemas/SortReq', param.sort); + const sort = await Sort.update(param.sortId, param.sort); T.emit('evt', { evt_type: 'sort:updated' }); return sort; } export async function sortCreate(param: { viewId: any; sort: SortReqType }) { + validatePayload('swagger.json#/components/schemas/SortReq', param.sort); + const sort = await Sort.insert({ ...param.sort, fk_view_id: param.viewId, diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts index 1ff534548f..fabf00b1c0 100644 --- a/packages/nocodb/src/lib/services/tableService.ts +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -9,6 +9,7 @@ import { UITypes, } from 'nocodb-sdk'; import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; +import { validatePayload } from '../meta/api/helpers'; import { NcError } from '../meta/helpers/catchError'; import getColumnPropsFromUIDT from '../meta/helpers/getColumnPropsFromUIDT'; import getColumnUiType from '../meta/helpers/getColumnUiType'; @@ -16,14 +17,16 @@ import getTableNameAlias, { getColumnNameAlias, } from '../meta/helpers/getTableName'; import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; -import Audit from '../models/Audit'; -import Column from '../models/Column'; -import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; -import Model from '../models/Model'; -import ModelRoleVisibility from '../models/ModelRoleVisibility'; -import Project from '../models/Project'; -import User from '../models/User'; -import View from '../models/View'; +import { + Audit, + Column, + LinkToAnotherRecordColumn, + Model, + ModelRoleVisibility, + Project, + User, + View, +} from '../models'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import { T } from 'nc-help'; @@ -304,6 +307,8 @@ export async function tableCreate(args: { table: TableReqType; user: User; }) { + validatePayload('swagger.json#/components/schemas/TableReq', args.table); + const project = await Project.getWithInfo(args.projectId); let base = project.bases[0]; diff --git a/packages/nocodb/src/lib/services/userService/index.ts b/packages/nocodb/src/lib/services/userService/index.ts index c18d35bd2e..d9e42bd651 100644 --- a/packages/nocodb/src/lib/services/userService/index.ts +++ b/packages/nocodb/src/lib/services/userService/index.ts @@ -13,6 +13,7 @@ import * as ejs from 'ejs'; import bcrypt from 'bcryptjs'; import { promisify } from 'util'; import { NC_APP_SETTINGS } from '../../constants'; +import { validatePayload } from '../../meta/api/helpers'; import { NcError } from '../../meta/helpers/catchError'; import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; import { Audit, Store, User } from '../../models'; @@ -79,6 +80,11 @@ export async function passwordChange(param: { user: UserType; req: any; }): Promise { + validatePayload( + 'swagger.json#/components/schemas/PasswordChangeReq', + param.body + ); + const { currentPassword, newPassword } = param.body; if (!currentPassword || !newPassword) { @@ -129,6 +135,11 @@ export async function passwordForgot(param: { siteUrl: string; req: any; }): Promise { + validatePayload( + 'swagger.json#/components/schemas/ForgotPasswordReq', + param.body + ); + const _email = param.body.email; if (!_email) { @@ -203,6 +214,11 @@ export async function passwordReset(param: { // todo: exclude req: any; }): Promise { + validatePayload( + 'swagger.json#/components/schemas/PasswordResetReq', + param.body + ); + const { token, body, req } = param; const user = await Noco.ncMeta.metaGet(null, null, MetaTable.USERS, { From fad9474b00921223de4b14a785c90a1368896a8b Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Mar 2023 10:50:20 +0530 Subject: [PATCH 41/64] fix: on ajv validation fail throw 400 bad request Signed-off-by: Pranav C --- packages/nocodb/src/lib/meta/helpers/catchError.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/meta/helpers/catchError.ts b/packages/nocodb/src/lib/meta/helpers/catchError.ts index f7d6f9dd4b..4c6da5d46d 100644 --- a/packages/nocodb/src/lib/meta/helpers/catchError.ts +++ b/packages/nocodb/src/lib/meta/helpers/catchError.ts @@ -412,7 +412,7 @@ export default function ( } else if (e instanceof NotImplemented) { return res.status(501).json({ msg: e.message }); } else if (e instanceof AjvError) { - return res.status(501).json({ msg: e.message, errors: e.errors }); + return res.status(400).json({ msg: e.message, errors: e.errors }); } next(e); } From 76e09b5dc0ac8e347d8fae82e28dc4f164d5636d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Mar 2023 14:40:01 +0530 Subject: [PATCH 42/64] refactor: move table update logic to service Signed-off-by: Pranav C --- .../src/lib/controllers/tableController.ts | 102 +---------------- .../nocodb/src/lib/services/tableService.ts | 103 ++++++++++++++++++ 2 files changed, 107 insertions(+), 98 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index fdf6528111..4494f6be24 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -66,105 +66,11 @@ export async function tableReorder(req: Request, res: Response) { // todo: move to table service export async function tableUpdate(req: Request, res) { - const model = await Model.get(req.params.tableId); - - const project = await Project.getWithInfo( - req.body.project_id || (req as any).ncProjectId - ); - const base = project.bases.find((b) => b.id === model.base_id); - - if (model.project_id !== project.id) { - NcError.badRequest('Model does not belong to project'); - } - - // if meta present update meta and return - // todo: allow user to update meta and other prop in single api call - if ('meta' in req.body) { - await Model.updateMeta(req.params.tableId, req.body.meta); - - return res.json({ msg: 'success' }); - } - - if (!req.body.table_name) { - NcError.badRequest( - 'Missing table name `table_name` property in request body' - ); - } - - if (base.is_meta && project.prefix) { - if (!req.body.table_name.startsWith(project.prefix)) { - req.body.table_name = `${project.prefix}${req.body.table_name}`; - } - } - - req.body.table_name = DOMPurify.sanitize(req.body.table_name); - - // validate table name - if (/^\s+|\s+$/.test(req.body.table_name)) { - NcError.badRequest( - 'Leading or trailing whitespace not allowed in table names' - ); - } - - if ( - !(await Model.checkTitleAvailable({ - table_name: req.body.table_name, - project_id: project.id, - base_id: base.id, - })) - ) { - NcError.badRequest('Duplicate table name'); - } - - if (!req.body.title) { - req.body.title = getTableNameAlias( - req.body.table_name, - project.prefix, - base - ); - } - - if ( - !(await Model.checkAliasAvailable({ - title: req.body.title, - project_id: project.id, - base_id: base.id, - })) - ) { - NcError.badRequest('Duplicate table alias'); - } - - const sqlMgr = await ProjectMgrv2.getSqlMgr(project); - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - - let tableNameLengthLimit = 255; - const sqlClientType = sqlClient.knex.clientType(); - if (sqlClientType === 'mysql2' || sqlClientType === 'mysql') { - tableNameLengthLimit = 64; - } else if (sqlClientType === 'pg') { - tableNameLengthLimit = 63; - } else if (sqlClientType === 'mssql') { - tableNameLengthLimit = 128; - } - - if (req.body.table_name.length > tableNameLengthLimit) { - NcError.badRequest(`Table name exceeds ${tableNameLengthLimit} characters`); - } - - await Model.updateAliasAndTableName( - req.params.tableId, - req.body.title, - req.body.table_name - ); - - await sqlMgr.sqlOpPlus(base, 'tableRename', { - ...req.body, - tn: req.body.table_name, - tn_old: model.table_name, + await tableService.tableUpdate({ + tableId: req.params.tableId, + table: req.body, + projectId: (req as any).ncProjectId, }); - - T.emit('evt', { evt_type: 'table:updated' }); - res.json({ msg: 'success' }); } diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts index fabf00b1c0..a4d63e61c9 100644 --- a/packages/nocodb/src/lib/services/tableService.ts +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -197,6 +197,109 @@ export async function tableDelete(param: { tableId: string; user: User }) { return table.delete(); } +export async function tableUpdate(param: { tableId: string; table: TableReqType; projectId?:string }) { + const model = await Model.get(param.tableId); + + const project = await Project.getWithInfo( + param.projectId || model.project_id + ); + const base = project.bases.find((b) => b.id === model.base_id); + + if (model.project_id !== project.id) { + NcError.badRequest('Model does not belong to project'); + } + + // if meta present update meta and return + // todo: allow user to update meta and other prop in single api call + if ('meta' in param.table) { + await Model.updateMeta(param.tableId, param.table.meta); + + return true; + } + + if (!param.table.table_name) { + NcError.badRequest( + 'Missing table name `table_name` property in request body' + ); + } + + if (base.is_meta && project.prefix) { + if (!param.table.table_name.startsWith(project.prefix)) { + param.table.table_name = `${project.prefix}${param.table.table_name}`; + } + } + + param.table.table_name = DOMPurify.sanitize(param.table.table_name); + + // validate table name + if (/^\s+|\s+$/.test(param.table.table_name)) { + NcError.badRequest( + 'Leading or trailing whitespace not allowed in table names' + ); + } + + if ( + !(await Model.checkTitleAvailable({ + table_name: param.table.table_name, + project_id: project.id, + base_id: base.id, + })) + ) { + NcError.badRequest('Duplicate table name'); + } + + if (!param.table.title) { + param.table.title = getTableNameAlias( + param.table.table_name, + project.prefix, + base + ); + } + + if ( + !(await Model.checkAliasAvailable({ + title: param.table.title, + project_id: project.id, + base_id: base.id, + })) + ) { + NcError.badRequest('Duplicate table alias'); + } + + const sqlMgr = await ProjectMgrv2.getSqlMgr(project); + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + + let tableNameLengthLimit = 255; + const sqlClientType = sqlClient.knex.clientType(); + if (sqlClientType === 'mysql2' || sqlClientType === 'mysql') { + tableNameLengthLimit = 64; + } else if (sqlClientType === 'pg') { + tableNameLengthLimit = 63; + } else if (sqlClientType === 'mssql') { + tableNameLengthLimit = 128; + } + + if (param.table.table_name.length > tableNameLengthLimit) { + NcError.badRequest(`Table name exceeds ${tableNameLengthLimit} characters`); + } + + await Model.updateAliasAndTableName( + param.tableId, + param.table.title, + param.table.table_name + ); + + await sqlMgr.sqlOpPlus(base, 'tableRename', { + ...param.table, + tn: param.table.table_name, + tn_old: model.table_name, + }); + + T.emit('evt', { evt_type: 'table:updated' }); + + return true; +} + export async function getTableWithAccessibleViews(param: { tableId: string; user: User; From 08130e82b1192a53e391b422eee4547b27917ff9 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Mar 2023 15:39:12 +0530 Subject: [PATCH 43/64] refactor: remove unused import statements Signed-off-by: Pranav C --- packages/nocodb/src/lib/controllers/tableController.ts | 8 -------- packages/nocodb/src/lib/services/tableService.ts | 6 +++++- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index 4494f6be24..9a3dc3c0cb 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -1,17 +1,9 @@ import { Request, Response, Router } from 'express'; -import DOMPurify from 'isomorphic-dompurify'; import { TableListType, TableReqType, TableType } from 'nocodb-sdk'; -import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import { NcError } from '../meta/helpers/catchError'; -import getTableNameAlias from '../meta/helpers/getTableName'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import Model from '../models/Model'; -import Project from '../models/Project'; -import { T } from 'nc-help'; import { tableService } from '../services'; -import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; export async function tableList(req: Request, res: Response) { res.json( diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts index a4d63e61c9..54d2d863b6 100644 --- a/packages/nocodb/src/lib/services/tableService.ts +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -197,7 +197,11 @@ export async function tableDelete(param: { tableId: string; user: User }) { return table.delete(); } -export async function tableUpdate(param: { tableId: string; table: TableReqType; projectId?:string }) { +export async function tableUpdate(param: { + tableId: string; + table: TableReqType; + projectId?: string; +}) { const model = await Model.get(param.tableId); const project = await Project.getWithInfo( From 2dec831b7e947cfc9eb883836c6a651921354fdd Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Mar 2023 17:25:59 +0530 Subject: [PATCH 44/64] refactor: include missing audit log data Signed-off-by: Pranav C --- .../src/lib/controllers/columnController.ts | 6 +- .../src/lib/controllers/tableController.ts | 1 + .../nocodb/src/lib/services/columnService.ts | 16 +++-- .../nocodb/src/lib/services/tableService.ts | 65 ++++++++++--------- 4 files changed, 50 insertions(+), 38 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/columnController.ts b/packages/nocodb/src/lib/controllers/columnController.ts index 8f6b34e12d..a6a230cfe1 100644 --- a/packages/nocodb/src/lib/controllers/columnController.ts +++ b/packages/nocodb/src/lib/controllers/columnController.ts @@ -16,6 +16,7 @@ export async function columnAdd( await columnService.columnAdd({ tableId: req.params.tableId, column: req.body, + req, }) ); } @@ -31,12 +32,15 @@ export async function columnUpdate(req: Request, res: Response) { await columnService.columnUpdate({ columnId: req.params.columnId, column: req.body, + req, }) ); } export async function columnDelete(req: Request, res: Response) { - res.json(await columnService.columnDelete({ columnId: req.params.columnId })); + res.json( + await columnService.columnDelete({ columnId: req.params.columnId, req }) + ); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index 9a3dc3c0cb..c59c8d4a13 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -42,6 +42,7 @@ export async function tableDelete(req: Request, res: Response) { const result = await tableService.tableDelete({ tableId: req.params.tableId, user: (req as any).session?.passport?.user, + req, }); res.json(result); diff --git a/packages/nocodb/src/lib/services/columnService.ts b/packages/nocodb/src/lib/services/columnService.ts index 185186d0a2..1609f02fe8 100644 --- a/packages/nocodb/src/lib/services/columnService.ts +++ b/packages/nocodb/src/lib/services/columnService.ts @@ -52,6 +52,7 @@ export enum Altered { } export async function columnUpdate(param: { + req?: any; columnId: string; column: ColumnReqType & { colOptions?: any }; cookie?: any; @@ -812,9 +813,9 @@ export async function columnUpdate(param: { project_id: base.project_id, op_type: AuditOperationTypes.TABLE_COLUMN, op_sub_type: AuditOperationSubTypes.UPDATED, - // user: (req as any)?.user?.email, + user: param.req?.user?.email, description: `updated column ${column.column_name} with alias ${column.title} from table ${table.table_name}`, - // ip: (req as any).clientIp, + ip: param.req?.clientIp, }).then(() => {}); await table.getColumns(); @@ -833,6 +834,7 @@ export async function columnSetAsPrimary(param: { columnId: string }) { } export async function columnAdd(param: { + req?: any; tableId: string; column: ColumnReqType; }) { @@ -1107,9 +1109,9 @@ export async function columnAdd(param: { project_id: base.project_id, op_type: AuditOperationTypes.TABLE_COLUMN, op_sub_type: AuditOperationSubTypes.CREATED, - // user: (req as any)?.user?.email, + user: param?.req?.user?.email, description: `created column ${colBody.column_name} with alias ${colBody.title} from table ${table.table_name}`, - // ip: (req as any).clientIp, + ip: param?.req.clientIp, }).then(() => {}); T.emit('evt', { evt_type: 'column:created' }); @@ -1117,7 +1119,7 @@ export async function columnAdd(param: { return table; } -export async function columnDelete(param: { columnId: string }) { +export async function columnDelete(param: { req?: any; columnId: string }) { const column = await Column.get({ colId: param.columnId }); const table = await Model.getWithInfo({ id: column.fk_model_id, @@ -1319,9 +1321,9 @@ export async function columnDelete(param: { columnId: string }) { project_id: base.project_id, op_type: AuditOperationTypes.TABLE_COLUMN, op_sub_type: AuditOperationSubTypes.DELETED, - // user: (req as any)?.user?.email, + user: param?.req?.user?.email, description: `deleted column ${column.column_name} with alias ${column.title} from table ${table.table_name}`, - // ip: (req as any).clientIp, + ip: param?.req.clientIp, }).then(() => {}); await table.getColumns(); diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts index 54d2d863b6..dec20a4a6b 100644 --- a/packages/nocodb/src/lib/services/tableService.ts +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -140,7 +140,11 @@ export function reorderTable(param: { tableId: string; order: any }) { return Model.updateOrder(param.tableId, param.order); } -export async function tableDelete(param: { tableId: string; user: User }) { +export async function tableDelete(param: { + tableId: string; + user: User; + req?: any; +}) { const table = await Model.getByIdOrName({ id: param.tableId }); await table.getColumns(); @@ -189,7 +193,7 @@ export async function tableDelete(param: { tableId: string; user: User }) { op_sub_type: AuditOperationSubTypes.DELETED, user: param.user?.email, description: `Deleted ${table.type} ${table.table_name} with alias ${table.title} `, - // ip: (req as any).clientIp, + ip: param.req?.clientIp, }).then(() => {}); T.emit('evt', { evt_type: 'table:deleted' }); @@ -408,24 +412,25 @@ export async function getAccessibleTables(param: { : (tableList.filter((t) => !t.mm) as Model[]); } -export async function tableCreate(args: { +export async function tableCreate(param: { projectId: string; baseId?: string; table: TableReqType; user: User; + req?: any; }) { - validatePayload('swagger.json#/components/schemas/TableReq', args.table); + validatePayload('swagger.json#/components/schemas/TableReq', param.table); - const project = await Project.getWithInfo(args.projectId); + const project = await Project.getWithInfo(param.projectId); let base = project.bases[0]; - if (args.baseId) { - base = project.bases.find((b) => b.id === args.baseId); + if (param.baseId) { + base = project.bases.find((b) => b.id === param.baseId); } if ( - !args.table.table_name || - (project.prefix && project.prefix === args.table.table_name) + !param.table.table_name || + (project.prefix && project.prefix === param.table.table_name) ) { NcError.badRequest( 'Missing table name `table_name` property in request body' @@ -433,15 +438,15 @@ export async function tableCreate(args: { } if (base.is_meta && project.prefix) { - if (!args.table.table_name.startsWith(project.prefix)) { - args.table.table_name = `${project.prefix}_${args.table.table_name}`; + if (!param.table.table_name.startsWith(project.prefix)) { + param.table.table_name = `${project.prefix}_${param.table.table_name}`; } } - args.table.table_name = DOMPurify.sanitize(args.table.table_name); + param.table.table_name = DOMPurify.sanitize(param.table.table_name); // validate table name - if (/^\s+|\s+$/.test(args.table.table_name)) { + if (/^\s+|\s+$/.test(param.table.table_name)) { NcError.badRequest( 'Leading or trailing whitespace not allowed in table names' ); @@ -449,7 +454,7 @@ export async function tableCreate(args: { if ( !(await Model.checkTitleAvailable({ - table_name: args.table.table_name, + table_name: param.table.table_name, project_id: project.id, base_id: base.id, })) @@ -457,9 +462,9 @@ export async function tableCreate(args: { NcError.badRequest('Duplicate table name'); } - if (!args.table.title) { - args.table.title = getTableNameAlias( - args.table.table_name, + if (!param.table.title) { + param.table.title = getTableNameAlias( + param.table.table_name, project.prefix, base ); @@ -467,7 +472,7 @@ export async function tableCreate(args: { if ( !(await Model.checkAliasAvailable({ - title: args.table.title, + title: param.table.title, project_id: project.id, base_id: base.id, })) @@ -489,13 +494,13 @@ export async function tableCreate(args: { tableNameLengthLimit = 128; } - if (args.table.table_name.length > tableNameLengthLimit) { + if (param.table.table_name.length > tableNameLengthLimit) { NcError.badRequest(`Table name exceeds ${tableNameLengthLimit} characters`); } const mxColumnLength = Column.getMaxColumnNameLength(sqlClientType); - for (const column of args.table.columns) { + for (const column of param.table.columns) { if (column.column_name.length > mxColumnLength) { NcError.badRequest( `Column name ${column.column_name} exceeds ${mxColumnLength} characters` @@ -503,13 +508,13 @@ export async function tableCreate(args: { } } - args.table.columns = args.table.columns?.map((c) => ({ + param.table.columns = param.table.columns?.map((c) => ({ ...getColumnPropsFromUIDT(c as any, base), cn: c.column_name, })); await sqlMgr.sqlOpPlus(base, 'tableCreate', { - ...args.table, - tn: args.table.table_name, + ...param.table, + tn: param.table.table_name, }); const columns: Array< @@ -517,7 +522,7 @@ export async function tableCreate(args: { cn: string; system?: boolean; } - > = (await sqlClient.columnList({ tn: args.table.table_name }))?.data?.list; + > = (await sqlClient.columnList({ tn: param.table.table_name }))?.data?.list; const tables = await Model.list({ project_id: project.id, @@ -529,20 +534,20 @@ export async function tableCreate(args: { base_id: base.id, op_type: AuditOperationTypes.TABLE, op_sub_type: AuditOperationSubTypes.CREATED, - user: args.user?.email, - description: `created table ${args.table.table_name} with alias ${args.table.title} `, - // ip: (req as any).clientIp, + user: param.user?.email, + description: `created table ${param.table.table_name} with alias ${param.table.title} `, + ip: param.req?.clientIp, }).then(() => {}); - mapDefaultDisplayValue(args.table.columns); + mapDefaultDisplayValue(param.table.columns); T.emit('evt', { evt_type: 'table:created' }); // todo: type correction const result = await Model.insert(project.id, base.id, { - ...args.table, + ...param.table, columns: columns.map((c, i) => { - const colMetaFromReq = args.table?.columns?.find( + const colMetaFromReq = param.table?.columns?.find( (c1) => c.cn === c1.column_name ); return { From b9cf291d212f411f7f27a774ed5405901287dd71 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Mar 2023 18:48:35 +0530 Subject: [PATCH 45/64] refactor: remove duplicate method Signed-off-by: Pranav C --- .../nocodb/src/lib/services/tableService.ts | 106 ------------------ 1 file changed, 106 deletions(-) diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts index dec20a4a6b..eb94a9c708 100644 --- a/packages/nocodb/src/lib/services/tableService.ts +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -201,112 +201,6 @@ export async function tableDelete(param: { return table.delete(); } -export async function tableUpdate(param: { - tableId: string; - table: TableReqType; - projectId?: string; -}) { - const model = await Model.get(param.tableId); - - const project = await Project.getWithInfo( - param.projectId || model.project_id - ); - const base = project.bases.find((b) => b.id === model.base_id); - - if (model.project_id !== project.id) { - NcError.badRequest('Model does not belong to project'); - } - - // if meta present update meta and return - // todo: allow user to update meta and other prop in single api call - if ('meta' in param.table) { - await Model.updateMeta(param.tableId, param.table.meta); - - return true; - } - - if (!param.table.table_name) { - NcError.badRequest( - 'Missing table name `table_name` property in request body' - ); - } - - if (base.is_meta && project.prefix) { - if (!param.table.table_name.startsWith(project.prefix)) { - param.table.table_name = `${project.prefix}${param.table.table_name}`; - } - } - - param.table.table_name = DOMPurify.sanitize(param.table.table_name); - - // validate table name - if (/^\s+|\s+$/.test(param.table.table_name)) { - NcError.badRequest( - 'Leading or trailing whitespace not allowed in table names' - ); - } - - if ( - !(await Model.checkTitleAvailable({ - table_name: param.table.table_name, - project_id: project.id, - base_id: base.id, - })) - ) { - NcError.badRequest('Duplicate table name'); - } - - if (!param.table.title) { - param.table.title = getTableNameAlias( - param.table.table_name, - project.prefix, - base - ); - } - - if ( - !(await Model.checkAliasAvailable({ - title: param.table.title, - project_id: project.id, - base_id: base.id, - })) - ) { - NcError.badRequest('Duplicate table alias'); - } - - const sqlMgr = await ProjectMgrv2.getSqlMgr(project); - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - - let tableNameLengthLimit = 255; - const sqlClientType = sqlClient.knex.clientType(); - if (sqlClientType === 'mysql2' || sqlClientType === 'mysql') { - tableNameLengthLimit = 64; - } else if (sqlClientType === 'pg') { - tableNameLengthLimit = 63; - } else if (sqlClientType === 'mssql') { - tableNameLengthLimit = 128; - } - - if (param.table.table_name.length > tableNameLengthLimit) { - NcError.badRequest(`Table name exceeds ${tableNameLengthLimit} characters`); - } - - await Model.updateAliasAndTableName( - param.tableId, - param.table.title, - param.table.table_name - ); - - await sqlMgr.sqlOpPlus(base, 'tableRename', { - ...param.table, - tn: param.table.table_name, - tn_old: model.table_name, - }); - - T.emit('evt', { evt_type: 'table:updated' }); - - return true; -} export async function getTableWithAccessibleViews(param: { tableId: string; From 94834ca58d04e944bc44e2a07786b10a45e887b1 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Fri, 3 Mar 2023 20:16:20 +0530 Subject: [PATCH 46/64] docs: updates to advanced edit menu options --- .../noco-docs/content/en/setup-and-usages/table-operations.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/noco-docs/content/en/setup-and-usages/table-operations.md b/packages/noco-docs/content/en/setup-and-usages/table-operations.md index 2aeeae8c43..f2b4b8ac61 100644 --- a/packages/noco-docs/content/en/setup-and-usages/table-operations.md +++ b/packages/noco-docs/content/en/setup-and-usages/table-operations.md @@ -70,7 +70,8 @@ After the click, it will show a menu and you can enter the column name and choos You can also click `Show more` for additional menu options. -image +![Screenshot 2023-03-03 at 8 13 07 PM](https://user-images.githubusercontent.com/86527202/222749857-0e793db2-a5d2-4b54-8d23-2a0cbbec8f5d.png) + Click `Save` button to create the new column. From af488e5ac557c23000221e5039256c2f1c43f265 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 00:05:58 +0530 Subject: [PATCH 47/64] refactor: remove old file Signed-off-by: Pranav C --- .../publicApis/publicDataExportApis.ts | 264 ------------------ 1 file changed, 264 deletions(-) delete mode 100644 packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts diff --git a/packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts b/packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts deleted file mode 100644 index 42ae1d91a6..0000000000 --- a/packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts +++ /dev/null @@ -1,264 +0,0 @@ -import { Request, Response, Router } from 'express'; -import * as XLSX from 'xlsx'; -import View from '../../models/View'; -import Model from '../../models/Model'; -import Base from '../../models/Base'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; -import { nocoExecute } from 'nc-help'; -import papaparse from 'papaparse'; -import { ErrorMessages, isSystemColumn, UITypes, ViewTypes } from 'nocodb-sdk'; -import Column from '../../models/Column'; -import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; -import LookupColumn from '../../models/LookupColumn'; -import catchError, { NcError } from '../../meta/helpers/catchError'; -import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; - -async function exportExcel(req: Request, res: Response) { - const view = await View.getByUUID(req.params.publicDataUuid); - if (!view) NcError.notFound('Not found'); - if ( - view.type !== ViewTypes.GRID && - view.type !== ViewTypes.KANBAN && - view.type !== ViewTypes.GALLERY && - view.type !== ViewTypes.MAP - ) - NcError.notFound('Not found'); - - if (view.password && view.password !== req.headers?.['xc-password']) { - NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); - } - - const model = await view.getModelWithInfo(); - - await view.getColumns(); - - const { offset, dbRows, elapsed } = await getDbRows(model, view, req); - - const fields = req.query.fields as string[]; - - const data = XLSX.utils.json_to_sheet( - dbRows.map((o: Record) => - Object.fromEntries(fields.map((f) => [f, o[f]])) - ), - { header: fields } - ); - - const wb = XLSX.utils.book_new(); - - XLSX.utils.book_append_sheet(wb, data, view.title); - - const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); - - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - view.title - )}-export.xlsx"`, - }); - res.end(buf); -} - -async function exportCsv(req: Request, res: Response) { - const view = await View.getByUUID(req.params.publicDataUuid); - const fields = req.query.fields; - - if (!view) NcError.notFound('Not found'); - if ( - view.type !== ViewTypes.GRID && - view.type !== ViewTypes.KANBAN && - view.type !== ViewTypes.GALLERY && - view.type !== ViewTypes.MAP - ) - NcError.notFound('Not found'); - - if (view.password && view.password !== req.headers?.['xc-password']) { - NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); - } - - const model = await view.getModelWithInfo(); - await view.getColumns(); - - const { offset, dbRows, elapsed } = await getDbRows(model, view, req); - - const data = papaparse.unparse( - { - fields: model.columns - .sort((c1, c2) => - Array.isArray(fields) - ? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any) - : 0 - ) - .filter( - (c) => - !fields || !Array.isArray(fields) || fields.includes(c.title as any) - ) - .map((c) => c.title), - data: dbRows, - }, - { - escapeFormulae: true, - } - ); - - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - view.title - )}-export.csv"`, - }); - res.send(data); -} - -async function getDbRows(model, view: View, req: Request) { - view.model.columns = view.columns - .filter((c) => c.show) - .map( - (c) => - new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any) - ) - .filter((column) => !isSystemColumn(column) || view.show_system_fields); - - if (!model) NcError.notFound('Table not found'); - - const listArgs: any = { ...req.query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - - const base = await Base.get(model.base_id); - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const requestObj = await getAst({ - query: req.query, - model, - view, - includePkByDefault: false, - }); - - let offset = +req.query.offset || 0; - const limit = 100; - // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; - const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; - const dbRows = []; - const startTime = process.hrtime(); - let elapsed, temp; - - for ( - elapsed = 0; - elapsed < timeout; - offset += limit, - temp = process.hrtime(startTime), - elapsed = temp[0] * 1000 + temp[1] / 1000000 - ) { - const rows = await nocoExecute( - requestObj, - await baseModel.list({ ...listArgs, offset, limit }), - {}, - listArgs - ); - - if (!rows?.length) { - offset = -1; - break; - } - - for (const row of rows) { - const dbRow = { ...row }; - - for (const column of view.model.columns) { - dbRow[column.title] = await serializeCellValue({ - value: row[column.title], - column, - }); - } - dbRows.push(dbRow); - } - } - return { offset, dbRows, elapsed }; -} - -async function serializeCellValue({ - value, - column, -}: { - column?: Column; - value: any; -}) { - if (!column) { - return value; - } - - if (!value) return value; - - switch (column?.uidt) { - case UITypes.Attachment: { - let data = value; - try { - if (typeof value === 'string') { - data = JSON.parse(value); - } - } catch {} - - return (data || []).map( - (attachment) => - `${encodeURI(attachment.title)}(${encodeURI(attachment.url)})` - ); - } - case UITypes.Lookup: - { - const colOptions = await column.getColOptions(); - const lookupColumn = await colOptions.getLookupColumn(); - return ( - await Promise.all( - [...(Array.isArray(value) ? value : [value])].map(async (v) => - serializeCellValue({ - value: v, - column: lookupColumn, - }) - ) - ) - ).join(', '); - } - break; - case UITypes.LinkToAnotherRecord: - { - const colOptions = - await column.getColOptions(); - const relatedModel = await colOptions.getRelatedTable(); - await relatedModel.getColumns(); - return [...(Array.isArray(value) ? value : [value])] - .map((v) => { - return v[relatedModel.displayValue?.title]; - }) - .join(', '); - } - break; - default: - if (value && typeof value === 'object') { - return JSON.stringify(value); - } - return value; - } -} - -const router = Router({ mergeParams: true }); -router.get( - '/api/v1/db/public/shared-view/:publicDataUuid/rows/export/csv', - catchError(exportCsv) -); -router.get( - '/api/v1/db/public/shared-view/:publicDataUuid/rows/export/excel', - catchError(exportExcel) -); -export default router; From 0efe4000e875d248211ed4d0d3fbee1f761c2668 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 00:06:30 +0530 Subject: [PATCH 48/64] refactor: move new implementation to public folder Signed-off-by: Pranav C --- .../controllers/{public => publicApis}/publicDataExportApis.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/nocodb/src/lib/controllers/{public => publicApis}/publicDataExportApis.ts (100%) diff --git a/packages/nocodb/src/lib/controllers/public/publicDataExportApis.ts b/packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/public/publicDataExportApis.ts rename to packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts From 7a5badcab5bd8497670dade74dad7c0b8edba9fd Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 00:07:20 +0530 Subject: [PATCH 49/64] refactor: use service in public api handlers Signed-off-by: Pranav C --- .../src/lib/controllers/publicApis/index.ts | 12 +- .../controllers/publicApis/publicDataApis.ts | 449 ++---------------- .../controllers/publicApis/publicMetaApis.ts | 99 +--- 3 files changed, 61 insertions(+), 499 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/publicApis/index.ts b/packages/nocodb/src/lib/controllers/publicApis/index.ts index 60028a421f..8e1c237b36 100644 --- a/packages/nocodb/src/lib/controllers/publicApis/index.ts +++ b/packages/nocodb/src/lib/controllers/publicApis/index.ts @@ -1,5 +1,9 @@ -import publicDataApis from './publicDataApis'; -import publicDataExportApis from './publicDataExportApis'; -import publicMetaApis from './publicMetaApis'; +import publicDataController from './publicDataApis'; +import publicDataExportController from './publicDataExportApis'; +import publicMetaController from './publicMetaApis'; -export { publicDataApis, publicDataExportApis, publicMetaApis }; +export { + publicDataController, + publicDataExportController, + publicMetaController, +}; diff --git a/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts b/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts index 792a6b4b61..46a73a0f2a 100644 --- a/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts +++ b/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts @@ -1,439 +1,72 @@ import { Request, Response, Router } from 'express'; -import Model from '../../models/Model'; -import { nocoExecute } from 'nc-help'; -import Base from '../../models/Base'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; -import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; -import View from '../../models/View'; -import catchError, { NcError } from '../../meta/helpers/catchError'; import multer from 'multer'; -import { ErrorMessages, UITypes, ViewTypes } from 'nocodb-sdk'; -import Column from '../../models/Column'; -import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; -import NcPluginMgrv2 from '../../meta/helpers/NcPluginMgrv2'; -import path from 'path'; -import { nanoid } from 'nanoid'; -import { mimeIcons } from '../../utils/mimeTypes'; -import slash from 'slash'; -import { sanitizeUrlPath } from '../../services/attachmentService'; -import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; -import { getColumnByIdOrName } from '../dataApis/helpers'; import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants'; +import catchError from '../../meta/helpers/catchError'; +import { publicDataService } from '../../services'; export async function dataList(req: Request, res: Response) { - try { - const view = await View.getByUUID(req.params.sharedViewUuid); - - if (!view) NcError.notFound('Not found'); - if ( - view.type !== ViewTypes.GRID && - view.type !== ViewTypes.KANBAN && - view.type !== ViewTypes.GALLERY && - view.type !== ViewTypes.MAP - ) { - NcError.notFound('Not found'); - } - - if (view.password && view.password !== req.headers?.['xc-password']) { - return NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); - } - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id, - }); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const listArgs: any = { ...req.query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - - let data = []; - let count = 0; - - try { - data = await nocoExecute( - await getAst({ - query: req.query, - model, - view, - }), - await baseModel.list(listArgs), - {}, - listArgs - ); - count = await baseModel.count(listArgs); - } catch (e) { - // show empty result instead of throwing error here - // e.g. search some text in a numeric field - } - - res.json({ - data: new PagedResponseImpl(data, { ...req.query, count }), - }); - } catch (e) { - console.log(e); - res.status(500).json({ msg: e.message }); - } + const pagedResponse = await publicDataService.dataList({ + query: req.query, + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + }); + res.json({ data: pagedResponse }); } // todo: Handle the error case where view doesnt belong to model async function groupedDataList(req: Request, res: Response) { - try { - const view = await View.getByUUID(req.params.sharedViewUuid); - - if (!view) NcError.notFound('Not found'); - - if ( - view.type !== ViewTypes.GRID && - view.type !== ViewTypes.KANBAN && - view.type !== ViewTypes.GALLERY - ) { - NcError.notFound('Not found'); - } - - if (view.password && view.password !== req.headers?.['xc-password']) { - return NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); - } - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id, - }); - - res.json(await getGroupedDataList(model, view, req)); - } catch (e) { - console.log(e); - res.status(500).json({ msg: e.message }); - } -} - -async function getGroupedDataList(model, view: View, req) { - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), + const groupedData = await publicDataService.groupedDataList({ + query: req.query, + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + groupColumnId: req.params.columnId, }); - - const requestObj = await getAst({ model, query: req.query, view }); - - const listArgs: any = { ...req.query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - try { - listArgs.options = JSON.parse(listArgs.optionsArrJson); - } catch (e) {} - - let data = []; - - try { - const groupedData = await baseModel.groupedList({ - ...listArgs, - groupColumnId: req.params.columnId, - }); - data = await nocoExecute( - { key: 1, value: requestObj }, - groupedData, - {}, - listArgs - ); - const countArr = await baseModel.groupedListCount({ - ...listArgs, - groupColumnId: req.params.columnId, - }); - data = data.map((item) => { - // todo: use map to avoid loop - const count = - countArr.find((countItem: any) => countItem.key === item.key)?.count ?? - 0; - - item.value = new PagedResponseImpl(item.value, { - ...req.query, - count: count, - }); - return item; - }); - } catch (e) { - // show empty result instead of throwing error here - // e.g. search some text in a numeric field - } - return data; + res.json(groupedData); } -async function dataInsert( - req: Request & { files: any[] }, - res: Response, - next -) { - const view = await View.getByUUID(req.params.sharedViewUuid); - - if (!view) return next(new Error('Not found')); - if (view.type !== ViewTypes.FORM) return next(new Error('Not found')); - - if (view.password && view.password !== req.headers?.['xc-password']) { - return res.status(403).json(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); - } - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id, +async function dataInsert(req: Request & { files: any[] }, res: Response) { + const insertResult = await publicDataService.dataInsert({ + sharedViewUuid: req.params.sharedViewUuid, + password: req.headers?.['xc-password'] as string, + body: req.body?.data, + siteUrl: (req as any).ncSiteUrl, + files: req.files, }); - const base = await Base.get(model.base_id); - const project = await base.getProject(); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - await view.getViewWithInfo(); - await view.getColumns(); - await view.getModelWithInfo(); - await view.model.getColumns(); - - const fields = (view.model.columns = view.columns - .filter((c) => c.show) - .reduce((o, c) => { - o[view.model.columnsById[c.fk_column_id].title] = new Column({ - ...c, - ...view.model.columnsById[c.fk_column_id], - } as any); - return o; - }, {}) as any); - - let body = req.body?.data; - - if (typeof body === 'string') body = JSON.parse(body); - - const insertObject = Object.entries(body).reduce((obj, [key, val]) => { - if (key in fields) { - obj[key] = val; - } - return obj; - }, {}); - - const attachments = {}; - const storageAdapter = await NcPluginMgrv2.storageAdapter(); - - for (const file of req.files || []) { - // remove `_` prefix and `[]` suffix - const fieldName = file?.fieldname?.replace(/^_|\[\d*]$/g, ''); - - const filePath = sanitizeUrlPath([ - 'v1', - project.title, - model.title, - fieldName, - ]); - if (fieldName in fields && fields[fieldName].uidt === UITypes.Attachment) { - attachments[fieldName] = attachments[fieldName] || []; - const fileName = `${nanoid(6)}_${file.originalname}`; - let url = await storageAdapter.fileCreate( - slash(path.join('nc', 'uploads', ...filePath, fileName)), - file - ); - - if (!url) { - url = `${(req as any).ncSiteUrl}/download/${filePath.join( - '/' - )}/${fileName}`; - } - - attachments[fieldName].push({ - url, - title: file.originalname, - mimetype: file.mimetype, - size: file.size, - icon: mimeIcons[path.extname(file.originalname).slice(1)] || undefined, - }); - } - } - - for (const [column, data] of Object.entries(attachments)) { - insertObject[column] = JSON.stringify(data); - } - - res.json(await baseModel.nestedInsert(insertObject, null)); + res.json(insertResult); } async function relDataList(req, res) { - const view = await View.getByUUID(req.params.sharedViewUuid); - - if (!view) NcError.notFound('Not found'); - if (view.type !== ViewTypes.FORM) NcError.notFound('Not found'); - - if (view.password && view.password !== req.headers?.['xc-password']) { - NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); - } - - const column = await Column.get({ colId: req.params.columnId }); - const colOptions = await column.getColOptions(); - - const model = await colOptions.getRelatedTable(); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const requestObj = await getAst({ + const pagedResponse = await publicDataService.relDataList({ query: req.query, - model, - extractOnlyPrimaries: true, + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + columnId: req.params.columnId, }); - let data = []; - let count = 0; - try { - data = data = await nocoExecute( - requestObj, - await baseModel.list(req.query), - {}, - req.query - ); - count = await baseModel.count(req.query); - } catch (e) { - // show empty result instead of throwing error here - // e.g. search some text in a numeric field - } - - res.json(new PagedResponseImpl(data, { ...req.query, count })); + res.json(pagedResponse); } export async function publicMmList(req: Request, res: Response) { - const view = await View.getByUUID(req.params.sharedViewUuid); - - if (!view) NcError.notFound('Not found'); - if (view.type !== ViewTypes.GRID) NcError.notFound('Not found'); - - if (view.password && view.password !== req.headers?.['xc-password']) { - NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); - } - - const column = await getColumnByIdOrName( - req.params.colId, - await view.getModel() - ); - - if (column.fk_model_id !== view.fk_model_id) - NcError.badRequest("Column doesn't belongs to the model"); - - const base = await Base.get(view.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: view.fk_model_id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const key = `List`; - const requestObj: any = { - [key]: 1, - }; - - const data = ( - await nocoExecute( - requestObj, - { - [key]: async (args) => { - return await baseModel.mmList( - { - colId: req.params.colId, - parentId: req.params.rowId, - }, - args - ); - }, - }, - {}, - - { nested: { [key]: req.query } } - ) - )?.[key]; - - const count: any = await baseModel.mmListCount({ - colId: req.params.colId, - parentId: req.params.rowId, + const paginatedResponse = await publicDataService.publicMmList({ + query: req.query, + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + columnId: req.params.columnId, + rowId: req.params.rowId, }); - - res.json(new PagedResponseImpl(data, { ...req.query, count })); + res.json(paginatedResponse); } export async function publicHmList(req: Request, res: Response) { - const view = await View.getByUUID(req.params.sharedViewUuid); - - if (!view) NcError.notFound('Not found'); - if (view.type !== ViewTypes.GRID) NcError.notFound('Not found'); - - if (view.password && view.password !== req.headers?.['xc-password']) { - NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); - } - - const column = await getColumnByIdOrName( - req.params.colId, - await view.getModel() - ); - - if (column.fk_model_id !== view.fk_model_id) - NcError.badRequest("Column doesn't belongs to the model"); - - const base = await Base.get(view.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: view.fk_model_id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const key = `List`; - const requestObj: any = { - [key]: 1, - }; - - const data = ( - await nocoExecute( - requestObj, - { - [key]: async (args) => { - return await baseModel.hmList( - { - colId: req.params.colId, - id: req.params.rowId, - }, - args - ); - }, - }, - {}, - { nested: { [key]: req.query } } - ) - )?.[key]; - - const count = await baseModel.hmListCount({ - colId: req.params.colId, - id: req.params.rowId, + const paginatedResponse = await publicDataService.publicHmList({ + query: req.query, + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, + columnId: req.params.columnId, + rowId: req.params.rowId, }); - - res.json(new PagedResponseImpl(data, { ...req.query, count })); + res.json(paginatedResponse); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/publicApis/publicMetaApis.ts b/packages/nocodb/src/lib/controllers/publicApis/publicMetaApis.ts index 7d5af6036f..238e3223ae 100644 --- a/packages/nocodb/src/lib/controllers/publicApis/publicMetaApis.ts +++ b/packages/nocodb/src/lib/controllers/publicApis/publicMetaApis.ts @@ -1,96 +1,21 @@ import { Request, Response, Router } from 'express'; -import catchError, { NcError } from '../../meta/helpers/catchError'; -import View from '../../models/View'; -import Model from '../../models/Model'; -import { - ErrorMessages, - LinkToAnotherRecordType, - RelationTypes, - UITypes, -} from 'nocodb-sdk'; -import Column from '../../models/Column'; -import Base from '../../models/Base'; -import Project from '../../models/Project'; -import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; +import catchError from '../../meta/helpers/catchError'; +import { publicMetaService } from '../../services'; export async function viewMetaGet(req: Request, res: Response) { - const view: View & { - relatedMetas?: { [ket: string]: Model }; - client?: string; - } = await View.getByUUID(req.params.sharedViewUuid); - - if (!view) NcError.notFound('Not found'); - - if (view.password && view.password !== req.headers?.['xc-password']) { - NcError.forbidden(ErrorMessages.INVALID_SHARED_VIEW_PASSWORD); - } - - await view.getFilters(); - await view.getSorts(); - - await view.getViewWithInfo(); - await view.getColumns(); - await view.getModelWithInfo(); - await view.model.getColumns(); - - const base = await Base.get(view.model.base_id); - view.client = base.type; - - // todo: return only required props - delete view['password']; - - view.model.columns = view.columns - .filter((c) => { - const column = view.model.columnsById[c.fk_column_id]; - return ( - c.show || - (column.rqd && !column.cdf && !column.ai) || - column.pk || - view.model.columns.some( - (c1) => - c1.uidt === UITypes.LinkToAnotherRecord && - (c1.colOptions).type === - RelationTypes.BELONGS_TO && - view.columns.some((vc) => vc.fk_column_id === c1.id && vc.show) && - (c1.colOptions).fk_child_column_id === - c.fk_column_id - ) - ); + res.json( + await publicMetaService.viewMetaGet({ + password: req.headers?.['xc-password'] as string, + sharedViewUuid: req.params.sharedViewUuid, }) - .map( - (c) => - new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any) - ) as any; - - const relatedMetas = {}; - - // load related table metas - for (const col of view.model.columns) { - if (UITypes.LinkToAnotherRecord === col.uidt) { - const colOpt = await col.getColOptions(); - relatedMetas[colOpt.fk_related_model_id] = await Model.getWithInfo({ - id: colOpt.fk_related_model_id, - }); - if (colOpt.type === 'mm') { - relatedMetas[colOpt.fk_mm_model_id] = await Model.getWithInfo({ - id: colOpt.fk_mm_model_id, - }); - } - } - } - - view.relatedMetas = relatedMetas; - - res.json(view); + ); } async function publicSharedBaseGet(req, res): Promise { - const project = await Project.getByUuid(req.params.sharedBaseUuid); - - if (!project) { - NcError.notFound(); - } - - res.json({ project_id: project.id }); + res.json( + await publicMetaService.publicSharedBaseGet({ + sharedBaseUuid: req.params.sharedBaseUuid, + }) + ); } const router = Router({ mergeParams: true }); From 4bb26cdf7e539032b339bfd2de6bcbe7f06a2a55 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 00:10:10 +0530 Subject: [PATCH 50/64] refactor: remove public folder Signed-off-by: Pranav C --- .../src/lib/controllers/public/index.ts | 9 -- .../lib/controllers/public/publicDataApis.ts | 105 ------------------ .../lib/controllers/public/publicMetaApis.ts | 31 ------ 3 files changed, 145 deletions(-) delete mode 100644 packages/nocodb/src/lib/controllers/public/index.ts delete mode 100644 packages/nocodb/src/lib/controllers/public/publicDataApis.ts delete mode 100644 packages/nocodb/src/lib/controllers/public/publicMetaApis.ts diff --git a/packages/nocodb/src/lib/controllers/public/index.ts b/packages/nocodb/src/lib/controllers/public/index.ts deleted file mode 100644 index 8e1c237b36..0000000000 --- a/packages/nocodb/src/lib/controllers/public/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -import publicDataController from './publicDataApis'; -import publicDataExportController from './publicDataExportApis'; -import publicMetaController from './publicMetaApis'; - -export { - publicDataController, - publicDataExportController, - publicMetaController, -}; diff --git a/packages/nocodb/src/lib/controllers/public/publicDataApis.ts b/packages/nocodb/src/lib/controllers/public/publicDataApis.ts deleted file mode 100644 index 46a73a0f2a..0000000000 --- a/packages/nocodb/src/lib/controllers/public/publicDataApis.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { Request, Response, Router } from 'express'; -import multer from 'multer'; -import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants'; -import catchError from '../../meta/helpers/catchError'; -import { publicDataService } from '../../services'; - -export async function dataList(req: Request, res: Response) { - const pagedResponse = await publicDataService.dataList({ - query: req.query, - password: req.headers?.['xc-password'] as string, - sharedViewUuid: req.params.sharedViewUuid, - }); - res.json({ data: pagedResponse }); -} - -// todo: Handle the error case where view doesnt belong to model -async function groupedDataList(req: Request, res: Response) { - const groupedData = await publicDataService.groupedDataList({ - query: req.query, - password: req.headers?.['xc-password'] as string, - sharedViewUuid: req.params.sharedViewUuid, - groupColumnId: req.params.columnId, - }); - res.json(groupedData); -} - -async function dataInsert(req: Request & { files: any[] }, res: Response) { - const insertResult = await publicDataService.dataInsert({ - sharedViewUuid: req.params.sharedViewUuid, - password: req.headers?.['xc-password'] as string, - body: req.body?.data, - siteUrl: (req as any).ncSiteUrl, - files: req.files, - }); - - res.json(insertResult); -} - -async function relDataList(req, res) { - const pagedResponse = await publicDataService.relDataList({ - query: req.query, - password: req.headers?.['xc-password'] as string, - sharedViewUuid: req.params.sharedViewUuid, - columnId: req.params.columnId, - }); - - res.json(pagedResponse); -} - -export async function publicMmList(req: Request, res: Response) { - const paginatedResponse = await publicDataService.publicMmList({ - query: req.query, - password: req.headers?.['xc-password'] as string, - sharedViewUuid: req.params.sharedViewUuid, - columnId: req.params.columnId, - rowId: req.params.rowId, - }); - res.json(paginatedResponse); -} - -export async function publicHmList(req: Request, res: Response) { - const paginatedResponse = await publicDataService.publicHmList({ - query: req.query, - password: req.headers?.['xc-password'] as string, - sharedViewUuid: req.params.sharedViewUuid, - columnId: req.params.columnId, - rowId: req.params.rowId, - }); - res.json(paginatedResponse); -} - -const router = Router({ mergeParams: true }); -router.get( - '/api/v1/db/public/shared-view/:sharedViewUuid/rows', - catchError(dataList) -); -router.get( - '/api/v1/db/public/shared-view/:sharedViewUuid/group/:columnId', - catchError(groupedDataList) -); -router.get( - '/api/v1/db/public/shared-view/:sharedViewUuid/nested/:columnId', - catchError(relDataList) -); -router.post( - '/api/v1/db/public/shared-view/:sharedViewUuid/rows', - multer({ - storage: multer.diskStorage({}), - limits: { - fieldSize: NC_ATTACHMENT_FIELD_SIZE, - }, - }).any(), - catchError(dataInsert) -); - -router.get( - '/api/v1/db/public/shared-view/:sharedViewUuid/rows/:rowId/mm/:colId', - catchError(publicMmList) -); -router.get( - '/api/v1/db/public/shared-view/:sharedViewUuid/rows/:rowId/hm/:colId', - catchError(publicHmList) -); - -export default router; diff --git a/packages/nocodb/src/lib/controllers/public/publicMetaApis.ts b/packages/nocodb/src/lib/controllers/public/publicMetaApis.ts deleted file mode 100644 index 238e3223ae..0000000000 --- a/packages/nocodb/src/lib/controllers/public/publicMetaApis.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Request, Response, Router } from 'express'; -import catchError from '../../meta/helpers/catchError'; -import { publicMetaService } from '../../services'; - -export async function viewMetaGet(req: Request, res: Response) { - res.json( - await publicMetaService.viewMetaGet({ - password: req.headers?.['xc-password'] as string, - sharedViewUuid: req.params.sharedViewUuid, - }) - ); -} -async function publicSharedBaseGet(req, res): Promise { - res.json( - await publicMetaService.publicSharedBaseGet({ - sharedBaseUuid: req.params.sharedBaseUuid, - }) - ); -} - -const router = Router({ mergeParams: true }); -router.get( - '/api/v1/db/public/shared-view/:sharedViewUuid/meta', - catchError(viewMetaGet) -); - -router.get( - '/api/v1/db/public/shared-base/:sharedBaseUuid/meta', - catchError(publicSharedBaseGet) -); -export default router; From f8b65c020c7d6208f18cad94b456e54042c4bf7d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 00:12:55 +0530 Subject: [PATCH 51/64] refactor: rename public folder name Signed-off-by: Pranav C --- .../{publicApis => publiController}/index.ts | 0 .../publicDataApis.ts | 0 .../publicDataExportApis.ts | 0 .../publicMetaApis.ts | 0 packages/nocodb/src/lib/meta/api/index.ts | 14 +++++++------- 5 files changed, 7 insertions(+), 7 deletions(-) rename packages/nocodb/src/lib/controllers/{publicApis => publiController}/index.ts (100%) rename packages/nocodb/src/lib/controllers/{publicApis => publiController}/publicDataApis.ts (100%) rename packages/nocodb/src/lib/controllers/{publicApis => publiController}/publicDataExportApis.ts (100%) rename packages/nocodb/src/lib/controllers/{publicApis => publiController}/publicMetaApis.ts (100%) diff --git a/packages/nocodb/src/lib/controllers/publicApis/index.ts b/packages/nocodb/src/lib/controllers/publiController/index.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publicApis/index.ts rename to packages/nocodb/src/lib/controllers/publiController/index.ts diff --git a/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts b/packages/nocodb/src/lib/controllers/publiController/publicDataApis.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts rename to packages/nocodb/src/lib/controllers/publiController/publicDataApis.ts diff --git a/packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts b/packages/nocodb/src/lib/controllers/publiController/publicDataExportApis.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publicApis/publicDataExportApis.ts rename to packages/nocodb/src/lib/controllers/publiController/publicDataExportApis.ts diff --git a/packages/nocodb/src/lib/controllers/publicApis/publicMetaApis.ts b/packages/nocodb/src/lib/controllers/publiController/publicMetaApis.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publicApis/publicMetaApis.ts rename to packages/nocodb/src/lib/controllers/publiController/publicMetaApis.ts diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index f1a676997a..a2ec3fecbe 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -43,10 +43,10 @@ import { oldDataApis, } from '../../controllers/dataApis'; import { - publicDataApis, - publicDataExportApis, - publicMetaApis, -} from '../../controllers/publicApis'; + publicDataController, + publicDataExportController, + publicMetaController, +} from '../../controllers/publiController'; import { Server, Socket } from 'socket.io'; import passport from 'passport'; @@ -81,9 +81,9 @@ export default function (router: Router, server) { router.use(viewColumnApis); router.use(gridViewApis); router.use(formViewColumnApis); - router.use(publicDataApis); - router.use(publicDataExportApis); - router.use(publicMetaApis); + router.use(publicDataController); + router.use(publicDataExportController); + router.use(publicMetaController); router.use(gridViewColumnApis); router.use(tableApis); router.use(galleryViewApis); From b72aed6b8cca3ce76cf6b948a39f7d0588c85e2f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 00:18:27 +0530 Subject: [PATCH 52/64] refactor: rename public folder and controller names Signed-off-by: Pranav C --- packages/nocodb/src/lib/controllers/publc/index.ts | 9 +++++++++ .../publicDataApis.ts => publc/publicDataController.ts} | 0 .../publicDataExportController.ts} | 0 .../publicMetaApis.ts => publc/publicMetaController.ts} | 0 .../nocodb/src/lib/controllers/publiController/index.ts | 9 --------- packages/nocodb/src/lib/meta/api/index.ts | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 packages/nocodb/src/lib/controllers/publc/index.ts rename packages/nocodb/src/lib/controllers/{publiController/publicDataApis.ts => publc/publicDataController.ts} (100%) rename packages/nocodb/src/lib/controllers/{publiController/publicDataExportApis.ts => publc/publicDataExportController.ts} (100%) rename packages/nocodb/src/lib/controllers/{publiController/publicMetaApis.ts => publc/publicMetaController.ts} (100%) delete mode 100644 packages/nocodb/src/lib/controllers/publiController/index.ts diff --git a/packages/nocodb/src/lib/controllers/publc/index.ts b/packages/nocodb/src/lib/controllers/publc/index.ts new file mode 100644 index 0000000000..1d63952955 --- /dev/null +++ b/packages/nocodb/src/lib/controllers/publc/index.ts @@ -0,0 +1,9 @@ +import publicDataController from './publicDataController'; +import publicDataExportController from './publicDataExportController'; +import publicMetaController from './publicMetaController'; + +export { + publicDataController, + publicDataExportController, + publicMetaController, +}; diff --git a/packages/nocodb/src/lib/controllers/publiController/publicDataApis.ts b/packages/nocodb/src/lib/controllers/publc/publicDataController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publiController/publicDataApis.ts rename to packages/nocodb/src/lib/controllers/publc/publicDataController.ts diff --git a/packages/nocodb/src/lib/controllers/publiController/publicDataExportApis.ts b/packages/nocodb/src/lib/controllers/publc/publicDataExportController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publiController/publicDataExportApis.ts rename to packages/nocodb/src/lib/controllers/publc/publicDataExportController.ts diff --git a/packages/nocodb/src/lib/controllers/publiController/publicMetaApis.ts b/packages/nocodb/src/lib/controllers/publc/publicMetaController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publiController/publicMetaApis.ts rename to packages/nocodb/src/lib/controllers/publc/publicMetaController.ts diff --git a/packages/nocodb/src/lib/controllers/publiController/index.ts b/packages/nocodb/src/lib/controllers/publiController/index.ts deleted file mode 100644 index 8e1c237b36..0000000000 --- a/packages/nocodb/src/lib/controllers/publiController/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -import publicDataController from './publicDataApis'; -import publicDataExportController from './publicDataExportApis'; -import publicMetaController from './publicMetaApis'; - -export { - publicDataController, - publicDataExportController, - publicMetaController, -}; diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index a2ec3fecbe..c96fe7d91d 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -46,7 +46,7 @@ import { publicDataController, publicDataExportController, publicMetaController, -} from '../../controllers/publiController'; +} from '../../controllers/publc'; import { Server, Socket } from 'socket.io'; import passport from 'passport'; From ac9596ded5b0a91821e0b0b5a69377ade52d7439 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 01:33:40 +0530 Subject: [PATCH 53/64] refactor: add conditional service function logic Signed-off-by: Pranav C --- .../src/lib/controllers/orgTokenController.ts | 12 +++++----- .../src/lib/meta/api/ee/orgTokenApis.ts | 22 ------------------- .../nocodb/src/lib/meta/helpers/getHandler.ts | 17 ++++++++++++++ .../src/lib/services/ee/orgTokenService.ts | 20 +++++++++++++++++ packages/nocodb/src/lib/services/index.ts | 3 ++- 5 files changed, 46 insertions(+), 28 deletions(-) delete mode 100644 packages/nocodb/src/lib/meta/api/ee/orgTokenApis.ts create mode 100644 packages/nocodb/src/lib/services/ee/orgTokenService.ts diff --git a/packages/nocodb/src/lib/controllers/orgTokenController.ts b/packages/nocodb/src/lib/controllers/orgTokenController.ts index 293ccd8c38..75e406fc42 100644 --- a/packages/nocodb/src/lib/controllers/orgTokenController.ts +++ b/packages/nocodb/src/lib/controllers/orgTokenController.ts @@ -1,13 +1,15 @@ import { Request, Response, Router } from 'express'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import getHandler from '../meta/helpers/getHandler'; +import { getConditionalHandler } from '../meta/helpers/getHandler'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { apiTokenListEE } from '../meta/api/ee/orgTokenApis'; -import { orgTokenService } from '../services'; +import { orgTokenService, orgTokenServiceEE } from '../services'; async function apiTokenList(req, res) { res.json( - await orgTokenService.apiTokenList({ + await getConditionalHandler( + orgTokenService.apiTokenList, + orgTokenServiceEE.apiTokenListEE + )({ query: req.query, user: req['user'], }) @@ -37,7 +39,7 @@ const router = Router({ mergeParams: true }); router.get( '/api/v1/tokens', metaApiMetrics, - ncMetaAclMw(getHandler(apiTokenList, apiTokenListEE), 'apiTokenList', { + ncMetaAclMw(apiTokenList, 'apiTokenList', { // allowedRoles: [OrgUserRoles.SUPER], blockApiTokenAccess: true, }) diff --git a/packages/nocodb/src/lib/meta/api/ee/orgTokenApis.ts b/packages/nocodb/src/lib/meta/api/ee/orgTokenApis.ts deleted file mode 100644 index 410d040c11..0000000000 --- a/packages/nocodb/src/lib/meta/api/ee/orgTokenApis.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { OrgUserRoles } from 'nocodb-sdk'; -import ApiToken from '../../../models/ApiToken'; -import { PagedResponseImpl } from '../../helpers/PagedResponse'; - -export async function apiTokenListEE(req, res) { - let fk_user_id = req.user.id; - - // if super admin get all tokens - if (req.user.roles.includes(OrgUserRoles.SUPER_ADMIN)) { - fk_user_id = undefined; - } - - res.json( - new PagedResponseImpl( - await ApiToken.listWithCreatedBy({ ...req.query, fk_user_id }), - { - ...req.query, - count: await ApiToken.count({}), - } - ) - ); -} diff --git a/packages/nocodb/src/lib/meta/helpers/getHandler.ts b/packages/nocodb/src/lib/meta/helpers/getHandler.ts index 1f40f45b2f..07c57a08eb 100644 --- a/packages/nocodb/src/lib/meta/helpers/getHandler.ts +++ b/packages/nocodb/src/lib/meta/helpers/getHandler.ts @@ -12,3 +12,20 @@ export default function getHandler( return eeHandler(...args); }; } + +export function getConditionalHandler< + T extends (...args: any[]) => any, + U extends (...args: any[]) => any +>( + defaultHandler: T, + eeHandler: U +): ( + ...args: Parameters | Parameters +) => Promise | ReturnType> { + return async (...args: Parameters | Parameters) => { + if (Noco.isEE()) { + return defaultHandler(...args); + } + return eeHandler(...args); + }; +} diff --git a/packages/nocodb/src/lib/services/ee/orgTokenService.ts b/packages/nocodb/src/lib/services/ee/orgTokenService.ts new file mode 100644 index 0000000000..9dd491144c --- /dev/null +++ b/packages/nocodb/src/lib/services/ee/orgTokenService.ts @@ -0,0 +1,20 @@ +import { OrgUserRoles, UserType } from 'nocodb-sdk'; +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import { ApiToken } from '../../models'; + +export async function apiTokenListEE(param: { user: UserType; query: any }) { + let fk_user_id = param.user.id; + + // if super admin get all tokens + if (param.user.roles.includes(OrgUserRoles.SUPER_ADMIN)) { + fk_user_id = undefined; + } + + return new PagedResponseImpl( + await ApiToken.listWithCreatedBy({ ...param.query, fk_user_id }), + { + ...(param.query || {}), + count: await ApiToken.count({}), + } + ); +} diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index f5ffe2f46f..f99398753c 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -21,7 +21,6 @@ export * as mapViewService from './mapViewService'; export * as modelVisibilityService from './modelVisibilityService'; export * as sharedBaseService from './sharedBaseService'; export * as orgUserService from './orgUserService'; -export * as orgTokenService from './orgTokenService'; export * as orgLicenseService from './orgLicenseService'; export * as projectUserService from './projectUserService'; export * as attachmentService from './attachmentService'; @@ -34,3 +33,5 @@ export * as swaggerService from './swaggerService'; export * as userService from './userService'; export * as syncService from './syncService'; export * from './public'; +export * as orgTokenService from './orgTokenService'; +export * as orgTokenServiceEE from './ee/orgTokenService'; From 4276f6da8ba6b1b5eefd7be4a2cbb5c637483665 Mon Sep 17 00:00:00 2001 From: navi Date: Sat, 4 Mar 2023 01:06:30 +0000 Subject: [PATCH 54/64] New translations en.json (Danish) --- packages/nc-gui/lang/da.json | 76 ++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/packages/nc-gui/lang/da.json b/packages/nc-gui/lang/da.json index 5a769dccb7..ff663368a2 100644 --- a/packages/nc-gui/lang/da.json +++ b/packages/nc-gui/lang/da.json @@ -1,6 +1,6 @@ { "general": { - "home": "Hjem", + "home": "Forside", "load": "Indlæs", "open": "Åbn", "close": "Luk", @@ -75,7 +75,7 @@ "hideField": "Skjul felt", "sortAsc": "Sortere stigende", "sortDesc": "Sortere nedadgående", - "geoDataField": "GeoData Field" + "geoDataField": "GeoData-felt" }, "objects": { "project": "Projekt", @@ -92,10 +92,10 @@ "records": "Optegnelser.", "webhook": "WebHook.", "webhooks": "Webhooks.", - "view": "Udsigt", - "views": "Visninger.", + "view": "Visning", + "views": "Visninger", "viewType": { - "grid": "Grid.", + "grid": "Grid", "gallery": "Galleri", "form": "Formular", "kanban": "Kanban.", @@ -217,7 +217,7 @@ "projName": "Projekt navn", "tableName": "Tabelnavn.", "viewName": "Se navn", - "viewLink": "Se Link.", + "viewLink": "Vis Link", "columnName": "Kolonne navn", "columnType": "Kolonne type", "roleName": "Rolle navn", @@ -235,7 +235,7 @@ "action": "Handling", "actions": "Handlinger", "operation": "Operation", - "operationSub": "Sub Operation", + "operationSub": "Underordnet operation", "operationType": "Driftstype", "operationSubType": "Drift Undertype", "description": "Beskrivelse", @@ -283,8 +283,8 @@ }, "docReference": "Dokumentreference.", "selectUserRole": "Vælg brugerrolle", - "childTable": "Børnebord", - "childColumn": "Barn kolonne", + "childTable": "Undertabel", + "childColumn": "Underkolonner", "linkToAnotherRecord": "Link til en anden post", "onUpdate": "På opdatering", "onDelete": "På Delete.", @@ -379,21 +379,21 @@ "reloadRoles": "Genindlæs roller", "nextPage": "Næste side", "prevPage": "Forrige side", - "nextRecord": "Næste rekord.", - "previousRecord": "Tidligere rekord.", + "nextRecord": "Næste post", + "previousRecord": "Forrige post", "copyApiURL": "COPY API URL.", - "createTable": "Tabel Create.", - "refreshTable": "Tabeller opdatere", - "renameTable": "Bord omdøb", - "deleteTable": "TABEL DELETE.", + "createTable": "Opret tabel", + "refreshTable": "Genopfrisk Tabeller", + "renameTable": "Omdøb Tabel", + "deleteTable": "Slet Tabel", "addField": "Tilføj nyt felt til denne tabel", "setDisplay": "Sæt som visningsværdi", - "addRow": "Tilføj ny række", - "saveRow": "Gem ro", + "addRow": "Tilføj ny post", + "saveRow": "Gem række", "saveAndExit": "Gem og afslutning", "saveAndStay": "Gem og bliv", "insertRow": "Indsæt ny række", - "deleteRow": "DELETE ROW.", + "deleteRow": "Slet Række", "duplicateRow": "Dupliker Række", "deleteSelectedRow": "Slet de valgte rækker", "importExcel": "Import Excel.", @@ -401,14 +401,14 @@ "downloadCSV": "Download som CSV.", "downloadExcel": "Download som XLSX", "uploadCSV": "Upload CSV.", - "import": "Importere", + "import": "Import", "importMetadata": "Import metadata.", "exportMetadata": "Eksport metadata.", "clearMetadata": "Klare metadata.", "exportToFile": "Eksporter til filer", "changePwd": "Skift kodeord", "createView": "Opret en visning", - "shareView": "Del View", + "shareView": "Del Visning", "listSharedView": "Shared View List.", "ListView": "Visninger List", "copyView": "Kopi visning", @@ -429,23 +429,23 @@ "importZip": "Import Zip.", "metaSync": "Synkroniser nu", "settings": "Indstillinger.", - "previewAs": "Forhåndsvisning så", - "resetReview": "Nulstil preview.", - "testDbConn": "Test Database Connection.", + "previewAs": "Forhåndsvisning som", + "resetReview": "Nulstil forhåndsvisning", + "testDbConn": "Test Database Forbindelse", "removeDbFromEnv": "Fjern databasen fra miljøet", "editConnJson": "Rediger forbindelse JSON", - "sponsorUs": "Sponsor os", + "sponsorUs": "Sponsorer os", "sendEmail": "SEND E-MAIL", "addUserToProject": "Tilføj bruger til projekt", "getApiSnippet": "Hent API-snippet", - "clearCell": "Klar celle", + "clearCell": "Ryd celle", "addFilterGroup": "Tilføj filtergruppe", "linkRecord": "Link record", "addNewRecord": "Tilføj ny post", "useConnectionUrl": "Brug forbindelses-URL", "toggleCommentsDraw": "Toggle kommentarer tegne", "expandRecord": "Udvid optegnelse", - "deleteRecord": "Slet registrering", + "deleteRecord": "Slet Post", "erd": { "showColumns": "Vis kolonner", "showPkAndFk": "Vis primære og fremmede nøgler", @@ -461,8 +461,8 @@ "addOrEditStack": "Tilføj / Rediger stak" }, "map": { - "mappedBy": "Mapped By", - "chooseMappingField": "Choose a Mapping Field" + "mappedBy": "Kortlagt af", + "chooseMappingField": "Vælg et kortlægningsfelt" } }, "tooltip": { @@ -530,9 +530,9 @@ "orgViewer": "Seeren har ikke lov til at oprette nye projekter, men kan få adgang til alle inviterede projekter." }, "map": { - "overLimit": "You're over the limit.", - "closeLimit": "You're getting close to the limit.", - "limitNumber": "The limit of markers shown in a Map View is 1000 records." + "overLimit": "Du er over grænsen.", + "closeLimit": "Du nærmer dig grænsen.", + "limitNumber": "Grænsen for antallet af markeringer, der vises i en kortvisning, er 1000 poster." }, "footerInfo": "Rækker per side", "upload": "Vælg fil for at uploade", @@ -616,7 +616,7 @@ "gallery": "Tilføj Gallery View.", "form": "Tilføj formularvisning", "kanban": "Tilføj Kanban View.", - "map": "Add Map View", + "map": "Tilføj Kortvisning", "calendar": "Tilføj kalendervisning" }, "tablesMetadataInSync": "Tabeller Metadata er synkroniseret", @@ -648,11 +648,11 @@ "deleteViewConfirmation": "Er du sikker på, at du vil slette denne visning?", "deleteTableConfirmation": "Ønsker du at slette tabellen", "showM2mTables": "Vis M2M-tabeller", - "showM2mTablesDesc": "Many-to-many relation is supported via a junction table & is hidden by default. Enable this option to list all such tables along with existing tables.", - "showNullInCells": "Show NULL in Cells", + "showM2mTablesDesc": "Mange-til-mange-relationer understøttes via en sammenknytnings-tabel (junction table) og er skjult som standard. Aktiver denne indstilling for at få vist alle sådanne tabeller sammen med eksisterende tabeller.", + "showNullInCells": "Vis NULL i celler", "showNullInCellsDesc": "Display 'NULL' tag in cells holding NULL value. This helps differentiate against cells holding EMPTY string.", - "showNullAndEmptyInFilter": "Show NULL and EMPTY in Filter", - "showNullAndEmptyInFilterDesc": "Enable 'additional' filters to differentiate fields containing NULL & Empty Strings. Default support for Blank treats both NULL & Empty strings alike.", + "showNullAndEmptyInFilter": "Vis NULL og EMPTY i Filter", + "showNullAndEmptyInFilterDesc": "Aktiver \"yderligere\" filtre for at skelne mellem felter, der indeholder NULL og tomme strenge. Standardunderstøttelse for Blank behandler både NULL- og tomme strenge ens.", "deleteKanbanStackConfirmation": "Hvis du sletter denne stak, fjernes også valgmuligheden `{stackToBeDeleted}` fra `{groupingField}`. Posterne vil blive flyttet til stakken \"uncategorized\".", "computedFieldEditWarning": "Beregnet felt: indholdet er skrivebeskyttet. Brug kolonne-redigeringsmenuen til at omkonfigurere", "computedFieldDeleteWarning": "Beregnet felt: indholdet er skrivebeskyttet. Det er ikke muligt at slette indholdet.", @@ -707,7 +707,7 @@ "nameShouldStartWithAnAlphabetOr_": "Navnet skal starte med et alfabet eller _", "followingCharactersAreNotAllowed": "Følgende tegn er ikke tilladt", "columnNameRequired": "Kolonnens navn er påkrævet", - "columnNameExceedsCharacters": "The length of column name exceeds the max {value} characters", + "columnNameExceedsCharacters": "Længden af kolonnenavnet overstiger maks. {value} tegn", "projectNameExceeds50Characters": "Projektnavnet overstiger 50 tegn", "projectNameCannotStartWithSpace": "Projektnavnet kan ikke begynde med et mellemrum", "requiredField": "Obligatorisk felt", @@ -740,7 +740,7 @@ }, "success": { "columnDuplicated": "Kolonne duplikeret med succes", - "rowDuplicatedWithoutSavedYet": "Row duplicated (not saved)", + "rowDuplicatedWithoutSavedYet": "Række duplikeret (ikke gemt)", "updatedUIACL": "Opdateret UI ACL for tabeller med succes", "pluginUninstalled": "Plugin afinstalleret med succes", "pluginSettingsSaved": "Plugin-indstillingerne er gemt med succes", From d0da5777f6fd54afd0d7e5ccdec58cff1e93f918 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 10:26:57 +0530 Subject: [PATCH 55/64] refactor: update data apis Signed-off-by: Pranav C --- .../controllers/dataApis/bulkDataAliasApis.ts | 94 ++- .../lib/controllers/dataApis/dataAliasApis.ts | 333 +++-------- .../dataApis/dataAliasExportApis.ts | 14 +- .../dataApis/dataAliasNestedApis.ts | 2 +- .../src/lib/controllers/dataApis/dataApis.ts | 566 +++--------------- .../controllers/{publc => public}/index.ts | 0 .../{publc => public}/publicDataController.ts | 0 .../publicDataExportController.ts | 0 .../{publc => public}/publicMetaController.ts | 0 packages/nocodb/src/lib/meta/api/index.ts | 2 +- .../src/lib/services/dataService/export.ts | 86 ++- 11 files changed, 242 insertions(+), 855 deletions(-) rename packages/nocodb/src/lib/controllers/{publc => public}/index.ts (100%) rename packages/nocodb/src/lib/controllers/{publc => public}/publicDataController.ts (100%) rename packages/nocodb/src/lib/controllers/{publc => public}/publicDataExportController.ts (100%) rename packages/nocodb/src/lib/controllers/{publc => public}/publicMetaController.ts (100%) diff --git a/packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasApis.ts b/packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasApis.ts index bef5276be6..968ab16165 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasApis.ts @@ -1,75 +1,69 @@ import { Request, Response, Router } from 'express'; -import { BaseModelSqlv2 } from '../../db/sql-data-mapper/lib/sql/BaseModelSqlv2'; -import Model from '../../models/Model'; -import Base from '../../models/Base'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { bulkDataService } from '../../services' import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import { getViewAndModelFromRequestByAliasOrId } from './helpers'; import apiMetrics from '../../meta/helpers/apiMetrics'; -type BulkOperation = - | 'bulkInsert' - | 'bulkUpdate' - | 'bulkUpdateAll' - | 'bulkDelete' - | 'bulkDeleteAll'; -async function getModelViewBase(req: Request) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - const base = await Base.get(model.base_id); - return { model, view, base }; -} - -async function executeBulkOperation( - req: Request, - res: Response, - operation: T, - options: Parameters -) { - const { model, view, base } = await getModelViewBase(req); - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - res.json(await baseModel[operation].apply(null, options)); -} async function bulkDataInsert(req: Request, res: Response) { - await executeBulkOperation(req, res, 'bulkInsert', [ - req.body, - { cookie: req }, - ]); + res.json( + await bulkDataService.bulkDataInsert({ + body: req.body, + cookie: req, + projectName: req.params.projectName, + tableName: req.params.tableName, + }) + ); } async function bulkDataUpdate(req: Request, res: Response) { - await executeBulkOperation(req, res, 'bulkUpdate', [ - req.body, - { cookie: req }, - ]); + res.json( + await bulkDataService.bulkDataUpdate({ + body: req.body, + cookie: req, + projectName: req.params.projectName, + tableName: req.params.tableName, + }) + ); } // todo: Integrate with filterArrJson bulkDataUpdateAll async function bulkDataUpdateAll(req: Request, res: Response) { - await executeBulkOperation(req, res, 'bulkUpdateAll', [ - req.query, - req.body, - { cookie: req }, - ]); + res.json( + await bulkDataService.bulkDataUpdateAll({ + body: req.body, + cookie: req, + projectName: req.params.projectName, + tableName: req.params.tableName, + query: req.query, + }) + ); } async function bulkDataDelete(req: Request, res: Response) { - await executeBulkOperation(req, res, 'bulkDelete', [ - req.body, - { cookie: req }, - ]); + res.json( + await bulkDataService.bulkDataDelete({ + body: req.body, + cookie: req, + projectName: req.params.projectName, + tableName: req.params.tableName, + }) + ); } // todo: Integrate with filterArrJson bulkDataDeleteAll async function bulkDataDeleteAll(req: Request, res: Response) { - await executeBulkOperation(req, res, 'bulkDeleteAll', [req.query]); + res.json( + await bulkDataService.bulkDataDeleteAll({ + // cookie: req, + projectName: req.params.projectName, + tableName: req.params.tableName, + query: req.query, + }) + ); } + + const router = Router({ mergeParams: true }); router.post( diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasApis.ts index a1bd604261..cfa35fafe6 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/dataAliasApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataAliasApis.ts @@ -1,304 +1,131 @@ import { Request, Response, Router } from 'express'; -import Model from '../../models/Model'; -import { nocoExecute } from 'nc-help'; -import Base from '../../models/Base'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; -import { NcError } from '../../meta/helpers/catchError'; -import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; -import View from '../../models/View'; +import { dataService } from '../../services' import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import { getViewAndModelFromRequestByAliasOrId } from './helpers'; import apiMetrics from '../../meta/helpers/apiMetrics'; -import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; import { parseHrtimeToSeconds } from '../../meta/api/helpers'; // todo: Handle the error case where view doesnt belong to model async function dataList(req: Request, res: Response) { const startTime = process.hrtime(); - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - const responseData = await getDataList(model, view, req); + const responseData = await dataService.dataList({ + query: req.query, + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + }); const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime)); res.setHeader('xc-db-response', elapsedSeconds); res.json(responseData); } async function dataFindOne(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - res.json(await getFindOne(model, view, req)); + res.json( + await dataService.dataFindOne({ + query: req.query, + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + }) + ); } async function dataGroupBy(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - res.json(await getDataGroupBy(model, view, req)); + res.json( + await dataService.dataGroupBy({ + query: req.query, + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + }) + ); } async function dataCount(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), + const countResult = await dataService.dataCount({ + query: req.query, + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, }); - const countArgs: any = { ...req.query }; - try { - countArgs.filterArr = JSON.parse(countArgs.filterArrJson); - } catch (e) {} - - const count = await baseModel.count(countArgs); - - res.json({ count }); + res.json(countResult); } -// todo: Handle the error case where view doesnt belong to model async function dataInsert(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - res.json(await baseModel.insert(req.body, null, req)); + res.json( + await dataService.dataInsert({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + body: req.body, + cookie: req, + }) + ); } async function dataUpdate(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - res.json(await baseModel.updateByPk(req.params.rowId, req.body, null, req)); + res.json( + await dataService.dataUpdate({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + body: req.body, + cookie: req, + rowId: req.params.rowId, + }) + ); } async function dataDelete(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - const base = await Base.get(model.base_id); - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - // todo: Should have error http status code - const message = await baseModel.hasLTARData(req.params.rowId, model); - if (message.length) { - res.json({ message }); - return; - } - res.json(await baseModel.delByPk(req.params.rowId, null, req)); -} - -async function getDataList(model, view: View, req) { - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const requestObj = await getAst({ model, query: req.query, view }); - - const listArgs: any = { ...req.query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - - let data = []; - let count = 0; - try { - data = await nocoExecute( - requestObj, - await baseModel.list(listArgs), - {}, - listArgs - ); - count = await baseModel.count(listArgs); - } catch (e) { - console.log(e); - NcError.internalServerError( - 'Internal Server Error, check server log for more details' - ); - } - - return new PagedResponseImpl(data, { - ...req.query, - count, - }); -} - -async function getFindOne(model, view: View, req) { - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const args: any = { ...req.query }; - try { - args.filterArr = JSON.parse(args.filterArrJson); - } catch (e) {} - try { - args.sortArr = JSON.parse(args.sortArrJson); - } catch (e) {} - - const data = await baseModel.findOne(args); - return data - ? await nocoExecute( - await getAst({ model, query: args, view }), - data, - {}, - {} - ) - : {}; -} - -async function getDataGroupBy(model, view: View, req) { - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const listArgs: any = { ...req.query }; - const data = await baseModel.groupBy({ ...req.query }); - const count = await baseModel.count(listArgs); - - return new PagedResponseImpl(data, { - ...req.query, - count, - }); + res.json( + await dataService.dataDelete({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + cookie: req, + rowId: req.params.rowId, + }) + ); } async function dataRead(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const row = await baseModel.readByPk(req.params.rowId); - - if (!row) { - NcError.notFound(); - } - res.json( - await nocoExecute( - await getAst({ model, query: req.query, view }), - row, - {}, - req.query - ) + await dataService.dataRead({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + rowId: req.params.rowId, + query: req.query, + }) ); } async function dataExist(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - res.json(await baseModel.exist(req.params.rowId)); + res.json( + await dataService.dataExist({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + rowId: req.params.rowId, + query: req.query, + }) + ); } // todo: Handle the error case where view doesnt belong to model async function groupedDataList(req: Request, res: Response) { const startTime = process.hrtime(); - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - const groupedData = await getGroupedDataList(model, view, req); + const groupedData = await dataService.groupedDataList({ + projectName: req.params.projectName, + tableName: req.params.tableName, + viewName: req.params.viewName, + query: req.query, + columnId: req.params.columnId, + }); const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime)); res.setHeader('xc-db-response', elapsedSeconds); res.json(groupedData); } - -async function getGroupedDataList(model, view: View, req) { - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const requestObj = await getAst({ model, query: req.query, view }); - - const listArgs: any = { ...req.query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - try { - listArgs.options = JSON.parse(listArgs.optionsArrJson); - } catch (e) {} - - let data = []; - // let count = 0 - try { - const groupedData = await baseModel.groupedList({ - ...listArgs, - groupColumnId: req.params.columnId, - }); - data = await nocoExecute( - { key: 1, value: requestObj }, - groupedData, - {}, - listArgs - ); - const countArr = await baseModel.groupedListCount({ - ...listArgs, - groupColumnId: req.params.columnId, - }); - data = data.map((item) => { - // todo: use map to avoid loop - const count = - countArr.find((countItem: any) => countItem.key === item.key)?.count ?? - 0; - - item.value = new PagedResponseImpl(item.value, { - ...req.query, - count: count, - }); - return item; - }); - } catch (e) { - console.log(e); - NcError.internalServerError( - 'Internal Server Error, check server log for more details' - ); - } - return data; -} - const router = Router({ mergeParams: true }); // table data crud apis diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts index c628e31814..d67f58f028 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts @@ -1,13 +1,13 @@ import { Request, Response, Router } from 'express'; import * as XLSX from 'xlsx'; +import apiMetrics from '../../meta/helpers/apiMetrics'; import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import { View } from '../../models'; import { extractCsvData, extractXlsxData, - getViewAndModelFromRequestByAliasOrId, -} from './helpers'; -import apiMetrics from '../../meta/helpers/apiMetrics'; -import View from '../../models/View'; +} from '../../services/dataService/helpers'; +import { getViewAndModelFromRequestByAliasOrId } from './helpers' async function excelDataExport(req: Request, res: Response) { const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); @@ -15,11 +15,7 @@ async function excelDataExport(req: Request, res: Response) { if (!targetView) { targetView = await View.getDefaultView(model.id); } - const { offset, elapsed, data } = await extractXlsxData({ - view: targetView, - query: req.query, - siteUrl: (req as any).ncSiteUrl, - }); + const { offset, elapsed, data } = await extractXlsxData(targetView, req); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, data, targetView.title); const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedApis.ts index 5151af03b7..3398bdffa3 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedApis.ts @@ -1,9 +1,9 @@ import { Request, Response, Router } from 'express'; import Model from '../../models/Model'; import Base from '../../models/Base'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2' import { getColumnByIdOrName, getViewAndModelFromRequestByAliasOrId, diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataApis.ts index 7c25607aab..fb5c35ab17 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/dataApis.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataApis.ts @@ -1,502 +1,116 @@ import { Request, Response, Router } from 'express'; -import Model from '../../models/Model'; -import { nocoExecute } from 'nc-help'; -import Base from '../../models/Base'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; -import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; -import View from '../../models/View'; +import { dataService } from '../../services'; import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import { NcError } from '../../meta/helpers/catchError'; import apiMetrics from '../../meta/helpers/apiMetrics'; -import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; - -export async function dataList(req: Request, res: Response, next) { - const view = await View.get(req.params.viewId); - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id || req.params.viewId, - }); - - if (!model) return next(new Error('Table not found')); - - res.json(await getDataList(model, view, req)); -} - -export async function mmList(req: Request, res: Response, next) { - const view = await View.get(req.params.viewId); - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id || req.params.viewId, - }); - - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const key = `${model.title}List`; - const requestObj: any = { - [key]: 1, - }; - - const data = ( - await nocoExecute( - requestObj, - { - [key]: async (args) => { - return await baseModel.mmList( - { - colId: req.params.colId, - parentId: req.params.rowId, - }, - args - ); - }, - }, - {}, - - { nested: { [key]: req.query } } - ) - )?.[key]; - - const count: any = await baseModel.mmListCount({ - colId: req.params.colId, - parentId: req.params.rowId, - }); +export async function dataList(req: Request, res: Response) { res.json( - new PagedResponseImpl(data, { - count, - ...req.query, + await dataService.dataListByViewId({ + viewId: req.params.viewId, + query: req.query, }) ); } -export async function mmExcludedList(req: Request, res: Response, next) { - const view = await View.get(req.params.viewId); - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id || req.params.viewId, - }); - - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const key = 'List'; - const requestObj: any = { - [key]: 1, - }; - - const data = ( - await nocoExecute( - requestObj, - { - [key]: async (args) => { - return await baseModel.getMmChildrenExcludedList( - { - colId: req.params.colId, - pid: req.params.rowId, - }, - args - ); - }, - }, - {}, - - { nested: { [key]: req.query } } - ) - )?.[key]; - - const count = await baseModel.getMmChildrenExcludedListCount( - { +export async function mmList(req: Request, res: Response) { + res.json( + await dataService.mmList({ + viewId: req.params.viewId, colId: req.params.colId, - pid: req.params.rowId, - }, - req.query + rowId: req.params.rowId, + query: req.query, + }) ); +} +export async function mmExcludedList(req: Request, res: Response) { res.json( - new PagedResponseImpl(data, { - count, - ...req.query, + await dataService.mmExcludedList({ + viewId: req.params.viewId, + colId: req.params.colId, + rowId: req.params.rowId, + query: req.query, }) ); } -export async function hmExcludedList(req: Request, res: Response, next) { - const view = await View.get(req.params.viewId); - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id || req.params.viewId, - }); - - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const key = 'List'; - const requestObj: any = { - [key]: 1, - }; - - const data = ( - await nocoExecute( - requestObj, - { - [key]: async (args) => { - return await baseModel.getHmChildrenExcludedList( - { - colId: req.params.colId, - pid: req.params.rowId, - }, - args - ); - }, - }, - {}, - - { nested: { [key]: req.query } } - ) - )?.[key]; - - const count = await baseModel.getHmChildrenExcludedListCount( - { +export async function hmExcludedList(req: Request, res: Response) { + res.json( + await dataService.hmExcludedList({ + viewId: req.params.viewId, colId: req.params.colId, - pid: req.params.rowId, - }, - req.query + rowId: req.params.rowId, + query: req.query, + }) ); +} +export async function btExcludedList(req: Request, res: Response) { res.json( - new PagedResponseImpl(data, { - count, - ...req.query, + await dataService.btExcludedList({ + viewId: req.params.viewId, + colId: req.params.colId, + rowId: req.params.rowId, + query: req.query, }) ); } -export async function btExcludedList(req: Request, res: Response, next) { - const view = await View.get(req.params.viewId); - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id || req.params.viewId, - }); - - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const key = 'List'; - const requestObj: any = { - [key]: 1, - }; - - const data = ( - await nocoExecute( - requestObj, - { - [key]: async (args) => { - return await baseModel.getBtChildrenExcludedList( - { - colId: req.params.colId, - cid: req.params.rowId, - }, - args - ); - }, - }, - {}, - - { nested: { [key]: req.query } } - ) - )?.[key]; - - const count = await baseModel.getBtChildrenExcludedListCount( - { +export async function hmList(req: Request, res: Response) { + res.json( + await dataService.hmList({ + viewId: req.params.viewId, colId: req.params.colId, - cid: req.params.rowId, - }, - req.query + rowId: req.params.rowId, + query: req.query, + }) ); +} +async function dataRead(req: Request, res: Response) { res.json( - new PagedResponseImpl(data, { - count, - ...req.query, + await dataService.dataReadByViewId({ + viewId: req.params.viewId, + rowId: req.params.rowId, + query: req.query, }) ); } -export async function hmList(req: Request, res: Response, next) { - const view = await View.get(req.params.viewId); - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id || req.params.viewId, - }); - - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const key = `${model.title}List`; - const requestObj: any = { - [key]: 1, - }; - - const data = ( - await nocoExecute( - requestObj, - { - [key]: async (args) => { - return await baseModel.hmList( - { - colId: req.params.colId, - id: req.params.rowId, - }, - args - ); - }, - }, - {}, - { nested: { [key]: req.query } } - ) - )?.[key]; - - const count = await baseModel.hmListCount({ - colId: req.params.colId, - id: req.params.rowId, - }); - +async function dataInsert(req: Request, res: Response) { res.json( - new PagedResponseImpl(data, { - totalRows: count, - } as any) + await dataService.dataInsertByViewId({ + viewId: req.params.viewId, + body: req.body, + cookie: req, + }) ); } -async function dataRead(req: Request, res: Response, next) { - try { - const model = await Model.getByIdOrName({ - id: req.params.viewId, - }); - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - res.json( - await nocoExecute( - await getAst({ model, query: req.query }), - await baseModel.readByPk(req.params.rowId), - {}, - {} - ) - ); - } catch (e) { - console.log(e); - NcError.internalServerError( - 'Internal Server Error, check server log for more details' - ); - } -} - -async function dataInsert(req: Request, res: Response, next) { - try { - const model = await Model.getByIdOrName({ - id: req.params.viewId, - }); - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - res.json(await baseModel.insert(req.body, null, req)); - } catch (e) { - console.log(e); - res.status(500).json({ msg: e.message }); - } -} - -// async function dataInsertNew(req: Request, res: Response) { -// const { model, view } = await getViewAndModelFromRequest(req); -// -// const base = await Base.get(model.base_id); -// -// const baseModel = await Model.getBaseModelSQL({ -// id: model.id, -// viewId: view?.id, -// dbDriver: NcConnectionMgrv2.get(base) -// }); -// -// res.json(await baseModel.insert(req.body, null, req)); -// } - -// async function dataUpdateNew(req: Request, res: Response) { -// const { model, view } = await getViewAndModelFromRequest(req); -// const base = await Base.get(model.base_id); -// -// const baseModel = await Model.getBaseModelSQL({ -// id: model.id, -// viewId: view.id, -// dbDriver: NcConnectionMgrv2.get(base) -// }); -// -// res.json(await baseModel.updateByPk(req.params.rowId, req.body, null, req)); -// } -async function dataUpdate(req: Request, res: Response, next) { - try { - const model = await Model.getByIdOrName({ - id: req.params.viewId, - }); - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - res.json(await baseModel.updateByPk(req.params.rowId, req.body, null, req)); - } catch (e) { - console.log(e); - res.status(500).json({ msg: e.message }); - } +async function dataUpdate(req: Request, res: Response) { + res.json( + await dataService.dataUpdateByViewId({ + viewId: req.params.viewId, + rowId: req.params.rowId, + body: req.body, + cookie: req, + }) + ); } -// -// async function dataDeleteNew(req: Request, res: Response) { -// const { model, view } = await getViewAndModelFromRequest(req); -// const base = await Base.get(model.base_id); -// const baseModel = await Model.getBaseModelSQL({ -// id: model.id, -// viewId: view.id, -// dbDriver: NcConnectionMgrv2.get(base) -// }); -// -// res.json(await baseModel.delByPk(req.params.rowId, null, req)); -// } -async function dataDelete(req: Request, res: Response, next) { - try { - const model = await Model.getByIdOrName({ - id: req.params.viewId, - }); - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - res.json(await baseModel.delByPk(req.params.rowId, null, req)); - } catch (e) { - console.log(e); - res.status(500).json({ msg: e.message }); - } +async function dataDelete(req: Request, res: Response) { + res.json( + await dataService.dataDeleteByViewId({ + viewId: req.params.viewId, + rowId: req.params.rowId, + cookie: req, + }) + ); } -async function getDataList(model, view: View, req) { - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const requestObj = await getAst({ query: req.query, model, view }); - - const listArgs: any = { ...req.query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - - let data = []; - let count = 0; - try { - data = await nocoExecute( - requestObj, - await baseModel.list(listArgs), - {}, - listArgs - ); - count = await baseModel.count(listArgs); - } catch (e) { - // show empty result instead of throwing error here - // e.g. search some text in a numeric field - console.log(e); - NcError.internalServerError( - 'Internal Server Error, check server log for more details' - ); - } - - return new PagedResponseImpl(data, { - count, - ...req.query, - }); -} -//@ts-ignore async function relationDataDelete(req, res) { - const view = await View.get(req.params.viewId); - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id || req.params.viewId, - }); - - if (!model) NcError.notFound('Table not found'); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - await baseModel.removeChild({ + await dataService.relationDataDelete({ + viewId: req.params.viewId, colId: req.params.colId, childId: req.params.childId, rowId: req.params.rowId, @@ -508,23 +122,8 @@ async function relationDataDelete(req, res) { //@ts-ignore async function relationDataAdd(req, res) { - const view = await View.get(req.params.viewId); - - const model = await Model.getByIdOrName({ - id: view?.fk_model_id || req.params.viewId, - }); - - if (!model) NcError.notFound('Table not found'); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - await baseModel.addChild({ + await dataService.relationDataAdd({ + viewId: req.params.viewId, colId: req.params.colId, childId: req.params.childId, rowId: req.params.rowId, @@ -536,25 +135,6 @@ async function relationDataAdd(req, res) { const router = Router({ mergeParams: true }); -// router.get('/data/:orgs/:projectName/:tableName',apiMetrics,ncMetaAclMw(dataListNew)); -// router.get( -// '/data/:orgs/:projectName/:tableName/views/:viewName', -// ncMetaAclMw(dataListNew) -// ); -// -// router.post( -// '/data/:orgs/:projectName/:tableName/views/:viewName', -// ncMetaAclMw(dataInsertNew) -// ); -// router.patch( -// '/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', -// ncMetaAclMw(dataUpdateNew) -// ); -// router.delete( -// '/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', -// ncMetaAclMw(dataDeleteNew) -// ); - router.get('/data/:viewId/', apiMetrics, ncMetaAclMw(dataList, 'dataList')); router.post( '/data/:viewId/', diff --git a/packages/nocodb/src/lib/controllers/publc/index.ts b/packages/nocodb/src/lib/controllers/public/index.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publc/index.ts rename to packages/nocodb/src/lib/controllers/public/index.ts diff --git a/packages/nocodb/src/lib/controllers/publc/publicDataController.ts b/packages/nocodb/src/lib/controllers/public/publicDataController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publc/publicDataController.ts rename to packages/nocodb/src/lib/controllers/public/publicDataController.ts diff --git a/packages/nocodb/src/lib/controllers/publc/publicDataExportController.ts b/packages/nocodb/src/lib/controllers/public/publicDataExportController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publc/publicDataExportController.ts rename to packages/nocodb/src/lib/controllers/public/publicDataExportController.ts diff --git a/packages/nocodb/src/lib/controllers/publc/publicMetaController.ts b/packages/nocodb/src/lib/controllers/public/publicMetaController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/publc/publicMetaController.ts rename to packages/nocodb/src/lib/controllers/public/publicMetaController.ts diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index c96fe7d91d..390d251bad 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -46,7 +46,7 @@ import { publicDataController, publicDataExportController, publicMetaController, -} from '../../controllers/publc'; +} from '../../controllers/public'; import { Server, Socket } from 'socket.io'; import passport from 'passport'; diff --git a/packages/nocodb/src/lib/services/dataService/export.ts b/packages/nocodb/src/lib/services/dataService/export.ts index f74e5299f9..53afe1ff85 100644 --- a/packages/nocodb/src/lib/services/dataService/export.ts +++ b/packages/nocodb/src/lib/services/dataService/export.ts @@ -1,22 +1,12 @@ -import { Request, Response, Router } from 'express'; -import { isSystemColumn } from 'nocodb-sdk'; -import * as XLSX from 'xlsx'; -import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; import { - extractXlsxData, - serializeCellValue, -} from '../../meta/api/dataApis/helpers'; -import { - extractCsvData, getViewAndModelByAliasOrId, - getViewAndModelFromRequestByAliasOrId, PathParams, } from './helpers'; -import apiMetrics from '../../helpers/apiMetrics'; -import View from '../../../models/View'; +import { View } from '../../models'; + -async function excelDataExport( +// Todo: bring logic from controller +export async function excelDataExport( param: PathParams & { query: any; } @@ -26,39 +16,39 @@ async function excelDataExport( if (!targetView) { targetView = await View.getDefaultView(model.id); } - const { offset, elapsed, data } = await extractXlsxData({ - view: targetView, - query: req.query, - }); - const wb = XLSX.utils.book_new(); - XLSX.utils.book_append_sheet(wb, data, targetView.title); - const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - targetView.title - )}-export.xlsx"`, - }); - res.end(buf); + // const { offset, elapsed, data } = await extractXlsxData({ + // view: targetView, + // query: param.query, + // }); + // const wb = XLSX.utils.book_new(); + // XLSX.utils.book_append_sheet(wb, data, targetView.title); + // const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); + // res.set({ + // 'Access-Control-Expose-Headers': 'nc-export-offset', + // 'nc-export-offset': offset, + // 'nc-export-elapsed-time': elapsed, + // 'Content-Disposition': `attachment; filename="${encodeURI( + // targetView.title + // )}-export.xlsx"`, + // }); + // res.end(buf); } -async function csvDataExport(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - let targetView = view; - if (!targetView) { - targetView = await View.getDefaultView(model.id); - } - const { offset, elapsed, data } = await extractCsvData(targetView, req); - - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - targetView.title - )}-export.csv"`, - }); - res.send(data); -} +// async function csvDataExport(req: Request, res: Response) { +// const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); +// let targetView = view; +// if (!targetView) { +// targetView = await View.getDefaultView(model.id); +// } +// const { offset, elapsed, data } = await extractCsvData(targetView, req); +// +// res.set({ +// 'Access-Control-Expose-Headers': 'nc-export-offset', +// 'nc-export-offset': offset, +// 'nc-export-elapsed-time': elapsed, +// 'Content-Disposition': `attachment; filename="${encodeURI( +// targetView.title +// )}-export.csv"`, +// }); +// res.send(data); +// } From 7c11883a105e0199ca26fec4189ea7859678b4e7 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 10:28:56 +0530 Subject: [PATCH 56/64] refactor: remove duplicate code Signed-off-by: Pranav C --- .../controllers/dataController/bulkData.ts | 91 ------ .../dataController/dataAliasNestedApis.ts | 291 ------------------ .../lib/controllers/dataController/export.ts | 75 ----- .../lib/controllers/dataController/index.ts | 261 ---------------- .../dataController/nestedAndDeprecated.ts | 192 ------------ 5 files changed, 910 deletions(-) delete mode 100644 packages/nocodb/src/lib/controllers/dataController/bulkData.ts delete mode 100644 packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts delete mode 100644 packages/nocodb/src/lib/controllers/dataController/export.ts delete mode 100644 packages/nocodb/src/lib/controllers/dataController/index.ts delete mode 100644 packages/nocodb/src/lib/controllers/dataController/nestedAndDeprecated.ts diff --git a/packages/nocodb/src/lib/controllers/dataController/bulkData.ts b/packages/nocodb/src/lib/controllers/dataController/bulkData.ts deleted file mode 100644 index 5f169d21e2..0000000000 --- a/packages/nocodb/src/lib/controllers/dataController/bulkData.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { Request, Response, Router } from 'express'; -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import apiMetrics from '../../meta/helpers/apiMetrics'; -import { bulkDataService } from '../../services'; - -async function bulkDataInsert(req: Request, res: Response) { - res.json( - await bulkDataService.bulkDataInsert({ - body: req.body, - cookie: req, - projectName: req.params.projectName, - tableName: req.params.tableName, - }) - ); -} - -async function bulkDataUpdate(req: Request, res: Response) { - res.json( - await bulkDataService.bulkDataUpdate({ - body: req.body, - cookie: req, - projectName: req.params.projectName, - tableName: req.params.tableName, - }) - ); -} - -// todo: Integrate with filterArrJson bulkDataUpdateAll -async function bulkDataUpdateAll(req: Request, res: Response) { - res.json( - await bulkDataService.bulkDataUpdateAll({ - body: req.body, - cookie: req, - projectName: req.params.projectName, - tableName: req.params.tableName, - query: req.query, - }) - ); -} - -async function bulkDataDelete(req: Request, res: Response) { - res.json( - await bulkDataService.bulkDataDelete({ - body: req.body, - cookie: req, - projectName: req.params.projectName, - tableName: req.params.tableName, - }) - ); -} - -// todo: Integrate with filterArrJson bulkDataDeleteAll -async function bulkDataDeleteAll(req: Request, res: Response) { - res.json( - await bulkDataService.bulkDataDeleteAll({ - // cookie: req, - projectName: req.params.projectName, - tableName: req.params.tableName, - query: req.query, - }) - ); -} -const router = Router({ mergeParams: true }); - -router.post( - '/api/v1/db/data/bulk/:orgs/:projectName/:tableName', - apiMetrics, - ncMetaAclMw(bulkDataInsert, 'bulkDataInsert') -); -router.patch( - '/api/v1/db/data/bulk/:orgs/:projectName/:tableName', - apiMetrics, - ncMetaAclMw(bulkDataUpdate, 'bulkDataUpdate') -); -router.patch( - '/api/v1/db/data/bulk/:orgs/:projectName/:tableName/all', - apiMetrics, - ncMetaAclMw(bulkDataUpdateAll, 'bulkDataUpdateAll') -); -router.delete( - '/api/v1/db/data/bulk/:orgs/:projectName/:tableName', - apiMetrics, - ncMetaAclMw(bulkDataDelete, 'bulkDataDelete') -); -router.delete( - '/api/v1/db/data/bulk/:orgs/:projectName/:tableName/all', - apiMetrics, - ncMetaAclMw(bulkDataDeleteAll, 'bulkDataDeleteAll') -); - -export default router; diff --git a/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts b/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts deleted file mode 100644 index d0dc296834..0000000000 --- a/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts +++ /dev/null @@ -1,291 +0,0 @@ -import { Request, Response, Router } from 'express'; -import Model from '../../models/Model'; -import Base from '../../models/Base'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; -import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import { - getColumnByIdOrName, - getViewAndModelFromRequestByAliasOrId, -} from '../dataApis/helpers'; -import { NcError } from '../../meta/helpers/catchError'; -import apiMetrics from '../../meta/helpers/apiMetrics'; - -// todo: handle case where the given column is not ltar -export async function mmList(req: Request, res: Response, next) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - - const data = await baseModel.mmList( - { - colId: column.id, - parentId: req.params.rowId, - }, - req.query as any - ); - const count: any = await baseModel.mmListCount({ - colId: column.id, - parentId: req.params.rowId, - }); - - res.json( - new PagedResponseImpl(data, { - count, - ...req.query, - }) - ); -} - -export async function mmExcludedList(req: Request, res: Response, next) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - const column = await getColumnByIdOrName(req.params.columnName, model); - - const data = await baseModel.getMmChildrenExcludedList( - { - colId: column.id, - pid: req.params.rowId, - }, - req.query - ); - - const count = await baseModel.getMmChildrenExcludedListCount( - { - colId: column.id, - pid: req.params.rowId, - }, - req.query - ); - - res.json( - new PagedResponseImpl(data, { - count, - ...req.query, - }) - ); -} - -export async function hmExcludedList(req: Request, res: Response, next) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - - const data = await baseModel.getHmChildrenExcludedList( - { - colId: column.id, - pid: req.params.rowId, - }, - req.query - ); - - const count = await baseModel.getHmChildrenExcludedListCount( - { - colId: column.id, - pid: req.params.rowId, - }, - req.query - ); - - res.json( - new PagedResponseImpl(data, { - count, - ...req.query, - }) - ); -} - -export async function btExcludedList(req: Request, res: Response, next) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - - const data = await baseModel.getBtChildrenExcludedList( - { - colId: column.id, - cid: req.params.rowId, - }, - req.query - ); - - const count = await baseModel.getBtChildrenExcludedListCount( - { - colId: column.id, - cid: req.params.rowId, - }, - req.query - ); - - res.json( - new PagedResponseImpl(data, { - count, - ...req.query, - }) - ); -} - -// todo: handle case where the given column is not ltar -export async function hmList(req: Request, res: Response, next) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - - const data = await baseModel.hmList( - { - colId: column.id, - id: req.params.rowId, - }, - req.query - ); - - const count = await baseModel.hmListCount({ - colId: column.id, - id: req.params.rowId, - }); - - res.json( - new PagedResponseImpl(data, { - count, - ...req.query, - } as any) - ); -} - -//@ts-ignore -async function relationDataRemove(req, res) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - if (!model) NcError.notFound('Table not found'); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - - await baseModel.removeChild({ - colId: column.id, - childId: req.params.refRowId, - rowId: req.params.rowId, - cookie: req, - }); - - res.json({ msg: 'success' }); -} - -//@ts-ignore -// todo: Give proper error message when reference row is already related and handle duplicate ref row id in hm -async function relationDataAdd(req, res) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - if (!model) NcError.notFound('Table not found'); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - await baseModel.addChild({ - colId: column.id, - childId: req.params.refRowId, - rowId: req.params.rowId, - cookie: req, - }); - - res.json({ msg: 'success' }); -} - -const router = Router({ mergeParams: true }); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/mm/:columnName/exclude', - apiMetrics, - ncMetaAclMw(mmExcludedList, 'mmExcludedList') -); -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/hm/:columnName/exclude', - apiMetrics, - ncMetaAclMw(hmExcludedList, 'hmExcludedList') -); -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/bt/:columnName/exclude', - apiMetrics, - ncMetaAclMw(btExcludedList, 'btExcludedList') -); - -router.post( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/:relationType/:columnName/:refRowId', - apiMetrics, - ncMetaAclMw(relationDataAdd, 'relationDataAdd') -); -router.delete( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/:relationType/:columnName/:refRowId', - apiMetrics, - ncMetaAclMw(relationDataRemove, 'relationDataRemove') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/mm/:columnName', - apiMetrics, - ncMetaAclMw(mmList, 'mmList') -); -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/hm/:columnName', - apiMetrics, - ncMetaAclMw(hmList, 'hmList') -); - -export default router; diff --git a/packages/nocodb/src/lib/controllers/dataController/export.ts b/packages/nocodb/src/lib/controllers/dataController/export.ts deleted file mode 100644 index 05f77dde5c..0000000000 --- a/packages/nocodb/src/lib/controllers/dataController/export.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { Request, Response, Router } from 'express'; -import * as XLSX from 'xlsx'; -import { getViewAndModelFromRequestByAliasOrId } from '../../meta/api/dataApis/helpers'; -import apiMetrics from '../../meta/helpers/apiMetrics'; -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import { View } from '../../models'; -import { - extractCsvData, - extractXlsxData, -} from '../../services/dataService/helpers'; - -async function excelDataExport(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - let targetView = view; - if (!targetView) { - targetView = await View.getDefaultView(model.id); - } - const { offset, elapsed, data } = await extractXlsxData(targetView, req); - const wb = XLSX.utils.book_new(); - XLSX.utils.book_append_sheet(wb, data, targetView.title); - const buf = XLSX.write(wb, { type: 'base64', bookType: 'xlsx' }); - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - targetView.title - )}-export.xlsx"`, - }); - res.end(buf); -} - -async function csvDataExport(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - let targetView = view; - if (!targetView) { - targetView = await View.getDefaultView(model.id); - } - const { offset, elapsed, data } = await extractCsvData(targetView, req); - - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - targetView.title - )}-export.csv"`, - }); - res.send(data); -} - -const router = Router({ mergeParams: true }); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/export/csv', - apiMetrics, - ncMetaAclMw(csvDataExport, 'exportCsv') -); -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/csv', - apiMetrics, - ncMetaAclMw(csvDataExport, 'exportCsv') -); -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/export/excel', - apiMetrics, - ncMetaAclMw(excelDataExport, 'exportExcel') -); -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/excel', - apiMetrics, - ncMetaAclMw(excelDataExport, 'exportExcel') -); - -export default router; diff --git a/packages/nocodb/src/lib/controllers/dataController/index.ts b/packages/nocodb/src/lib/controllers/dataController/index.ts deleted file mode 100644 index 522ed1be5e..0000000000 --- a/packages/nocodb/src/lib/controllers/dataController/index.ts +++ /dev/null @@ -1,261 +0,0 @@ -import { Request, Response, Router } from 'express'; -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import apiMetrics from '../../meta/helpers/apiMetrics'; -import { parseHrtimeToSeconds } from '../../meta/api/helpers'; - -import { dataService } from '../../services'; - -// todo: Handle the error case where view doesnt belong to model -async function dataList(req: Request, res: Response) { - const startTime = process.hrtime(); - const responseData = await dataService.dataList({ - query: req.query, - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - }); - const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime)); - res.setHeader('xc-db-response', elapsedSeconds); - res.json(responseData); -} - -async function dataFindOne(req: Request, res: Response) { - res.json( - await dataService.dataFindOne({ - query: req.query, - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - }) - ); -} - -async function dataGroupBy(req: Request, res: Response) { - res.json( - await dataService.dataGroupBy({ - query: req.query, - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - }) - ); -} - -async function dataCount(req: Request, res: Response) { - const countResult = await dataService.dataCount({ - query: req.query, - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - }); - - res.json(countResult); -} - -async function dataInsert(req: Request, res: Response) { - res.json( - await dataService.dataInsert({ - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - body: req.body, - cookie: req, - }) - ); -} - -async function dataUpdate(req: Request, res: Response) { - res.json( - await dataService.dataUpdate({ - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - body: req.body, - cookie: req, - rowId: req.params.rowId, - }) - ); -} - -async function dataDelete(req: Request, res: Response) { - res.json( - await dataService.dataDelete({ - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - cookie: req, - rowId: req.params.rowId, - }) - ); -} - -async function dataRead(req: Request, res: Response) { - res.json( - await dataService.dataRead({ - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - rowId: req.params.rowId, - query: req.query, - }) - ); -} - -async function dataExist(req: Request, res: Response) { - res.json( - await dataService.dataExist({ - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - rowId: req.params.rowId, - query: req.query, - }) - ); -} - -// todo: Handle the error case where view doesnt belong to model -async function groupedDataList(req: Request, res: Response) { - const startTime = process.hrtime(); - const groupedData = await dataService.groupedDataList({ - projectName: req.params.projectName, - tableName: req.params.tableName, - viewName: req.params.viewName, - query: req.query, - columnId: req.params.columnId, - }); - const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime)); - res.setHeader('xc-db-response', elapsedSeconds); - res.json(groupedData); -} - -const router = Router({ mergeParams: true }); - -// table data crud apis -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName', - apiMetrics, - ncMetaAclMw(dataList, 'dataList') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/find-one', - apiMetrics, - ncMetaAclMw(dataFindOne, 'dataFindOne') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/groupby', - apiMetrics, - ncMetaAclMw(dataGroupBy, 'dataGroupBy') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/group/:columnId', - apiMetrics, - ncMetaAclMw(groupedDataList, 'groupedDataList') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/exist', - apiMetrics, - ncMetaAclMw(dataExist, 'dataExist') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/count', - apiMetrics, - ncMetaAclMw(dataCount, 'dataCount') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/count', - apiMetrics, - ncMetaAclMw(dataCount, 'dataCount') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId', - apiMetrics, - ncMetaAclMw(dataRead, 'dataRead') -); - -router.patch( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId', - apiMetrics, - ncMetaAclMw(dataUpdate, 'dataUpdate') -); - -router.delete( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId', - apiMetrics, - ncMetaAclMw(dataDelete, 'dataDelete') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName', - apiMetrics, - ncMetaAclMw(dataList, 'dataList') -); - -// table view data crud apis -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName', - apiMetrics, - ncMetaAclMw(dataList, 'dataList') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/find-one', - apiMetrics, - ncMetaAclMw(dataFindOne, 'dataFindOne') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/groupby', - apiMetrics, - ncMetaAclMw(dataGroupBy, 'dataGroupBy') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/group/:columnId', - apiMetrics, - ncMetaAclMw(groupedDataList, 'groupedDataList') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId/exist', - apiMetrics, - ncMetaAclMw(dataExist, 'dataExist') -); - -router.post( - '/api/v1/db/data/:orgs/:projectName/:tableName', - apiMetrics, - ncMetaAclMw(dataInsert, 'dataInsert') -); - -router.post( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName', - apiMetrics, - ncMetaAclMw(dataInsert, 'dataInsert') -); - -router.patch( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', - apiMetrics, - ncMetaAclMw(dataUpdate, 'dataUpdate') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', - apiMetrics, - ncMetaAclMw(dataRead, 'dataRead') -); - -router.delete( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', - apiMetrics, - ncMetaAclMw(dataDelete, 'dataDelete') -); - -export default router; diff --git a/packages/nocodb/src/lib/controllers/dataController/nestedAndDeprecated.ts b/packages/nocodb/src/lib/controllers/dataController/nestedAndDeprecated.ts deleted file mode 100644 index 6914383321..0000000000 --- a/packages/nocodb/src/lib/controllers/dataController/nestedAndDeprecated.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { Request, Response, Router } from 'express'; -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import apiMetrics from '../../meta/helpers/apiMetrics'; -import { dataService } from '../../services'; - -export async function dataList(req: Request, res: Response) { - res.json( - await dataService.dataListByViewId({ - viewId: req.params.viewId, - query: req.query, - }) - ); -} - -export async function mmList(req: Request, res: Response) { - res.json( - await dataService.mmList({ - viewId: req.params.viewId, - colId: req.params.colId, - rowId: req.params.rowId, - query: req.query, - }) - ); -} - -export async function mmExcludedList(req: Request, res: Response) { - res.json( - await dataService.mmExcludedList({ - viewId: req.params.viewId, - colId: req.params.colId, - rowId: req.params.rowId, - query: req.query, - }) - ); -} - -export async function hmExcludedList(req: Request, res: Response) { - res.json( - await dataService.hmExcludedList({ - viewId: req.params.viewId, - colId: req.params.colId, - rowId: req.params.rowId, - query: req.query, - }) - ); -} - -export async function btExcludedList(req: Request, res: Response) { - res.json( - await dataService.btExcludedList({ - viewId: req.params.viewId, - colId: req.params.colId, - rowId: req.params.rowId, - query: req.query, - }) - ); -} - -export async function hmList(req: Request, res: Response) { - res.json( - await dataService.hmList({ - viewId: req.params.viewId, - colId: req.params.colId, - rowId: req.params.rowId, - query: req.query, - }) - ); -} - -async function dataRead(req: Request, res: Response) { - res.json( - await dataService.dataReadByViewId({ - viewId: req.params.viewId, - rowId: req.params.rowId, - query: req.query, - }) - ); -} - -async function dataInsert(req: Request, res: Response) { - res.json( - await dataService.dataInsertByViewId({ - viewId: req.params.viewId, - body: req.body, - cookie: req, - }) - ); -} - -async function dataUpdate(req: Request, res: Response) { - res.json( - await dataService.dataUpdateByViewId({ - viewId: req.params.viewId, - rowId: req.params.rowId, - body: req.body, - cookie: req, - }) - ); -} - -async function dataDelete(req: Request, res: Response) { - res.json( - await dataService.dataDeleteByViewId({ - viewId: req.params.viewId, - rowId: req.params.rowId, - cookie: req, - }) - ); -} - -async function relationDataDelete(req, res) { - await dataService.relationDataDelete({ - viewId: req.params.viewId, - colId: req.params.colId, - childId: req.params.childId, - rowId: req.params.rowId, - cookie: req, - }); - - res.json({ msg: 'success' }); -} - -//@ts-ignore -async function relationDataAdd(req, res) { - await dataService.relationDataAdd({ - viewId: req.params.viewId, - colId: req.params.colId, - childId: req.params.childId, - rowId: req.params.rowId, - cookie: req, - }); - - res.json({ msg: 'success' }); -} - -const router = Router({ mergeParams: true }); - -router.get('/data/:viewId/', apiMetrics, ncMetaAclMw(dataList, 'dataList')); -router.post( - '/data/:viewId/', - apiMetrics, - ncMetaAclMw(dataInsert, 'dataInsert') -); -router.get( - '/data/:viewId/:rowId', - apiMetrics, - ncMetaAclMw(dataRead, 'dataRead') -); -router.patch( - '/data/:viewId/:rowId', - apiMetrics, - ncMetaAclMw(dataUpdate, 'dataUpdate') -); -router.delete( - '/data/:viewId/:rowId', - apiMetrics, - ncMetaAclMw(dataDelete, 'dataDelete') -); - -router.get( - '/data/:viewId/:rowId/mm/:colId', - apiMetrics, - ncMetaAclMw(mmList, 'mmList') -); -router.get( - '/data/:viewId/:rowId/hm/:colId', - apiMetrics, - ncMetaAclMw(hmList, 'hmList') -); - -router.get( - '/data/:viewId/:rowId/mm/:colId/exclude', - ncMetaAclMw(mmExcludedList, 'mmExcludedList') -); -router.get( - '/data/:viewId/:rowId/hm/:colId/exclude', - ncMetaAclMw(hmExcludedList, 'hmExcludedList') -); -router.get( - '/data/:viewId/:rowId/bt/:colId/exclude', - ncMetaAclMw(btExcludedList, 'btExcludedList') -); - -router.post( - '/data/:viewId/:rowId/:relationType/:colId/:childId', - ncMetaAclMw(relationDataAdd, 'relationDataAdd') -); -router.delete( - '/data/:viewId/:rowId/:relationType/:colId/:childId', - ncMetaAclMw(relationDataDelete, 'relationDataDelete') -); -export default router; From aabeacf224d22fbf6bb031c7bf926769c8db0550 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 10:47:23 +0530 Subject: [PATCH 57/64] refactor: rename file names Signed-off-by: Pranav C --- ...liasApis.ts => bulkDataAliasController.ts} | 0 ...ataAliasApis.ts => dataAliasController.ts} | 0 ...rtApis.ts => dataAliasExportController.ts} | 0 ...edApis.ts => dataAliasNestedController.ts} | 0 .../{dataApis.ts => dataController.ts} | 0 .../src/lib/controllers/dataApis/index.ts | 24 +-- .../{oldDataApis.ts => oldDataController.ts} | 0 packages/nocodb/src/lib/meta/api/index.ts | 164 +++++++++--------- 8 files changed, 94 insertions(+), 94 deletions(-) rename packages/nocodb/src/lib/controllers/dataApis/{bulkDataAliasApis.ts => bulkDataAliasController.ts} (100%) rename packages/nocodb/src/lib/controllers/dataApis/{dataAliasApis.ts => dataAliasController.ts} (100%) rename packages/nocodb/src/lib/controllers/dataApis/{dataAliasExportApis.ts => dataAliasExportController.ts} (100%) rename packages/nocodb/src/lib/controllers/dataApis/{dataAliasNestedApis.ts => dataAliasNestedController.ts} (100%) rename packages/nocodb/src/lib/controllers/dataApis/{dataApis.ts => dataController.ts} (100%) rename packages/nocodb/src/lib/controllers/dataApis/{oldDataApis.ts => oldDataController.ts} (100%) diff --git a/packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasApis.ts b/packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasController.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/dataAliasApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/dataAliasController.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/dataAliasExportApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/dataAliasExportController.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedController.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataApis.ts b/packages/nocodb/src/lib/controllers/dataApis/dataController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/dataApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/dataController.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/index.ts b/packages/nocodb/src/lib/controllers/dataApis/index.ts index f7b699f340..122ec7fac2 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/index.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/index.ts @@ -1,15 +1,15 @@ -import dataApis from './dataApis'; -import oldDataApis from './oldDataApis'; -import dataAliasApis from './dataAliasApis'; -import bulkDataAliasApis from './bulkDataAliasApis'; -import dataAliasNestedApis from '../../controllers/dataController/dataAliasNestedApis'; -import dataAliasExportApis from './dataAliasExportApis'; +import dataController from './dataController'; +import oldDataController from './oldDataController'; +import dataAliasController from './dataAliasController'; +import bulkDataAliasController from './bulkDataAliasController'; +import dataAliasNestedController from './dataAliasNestedController'; +import dataAliasExportController from './dataAliasExportController'; export { - dataApis, - oldDataApis, - dataAliasApis, - bulkDataAliasApis, - dataAliasNestedApis, - dataAliasExportApis, + dataController, + oldDataController, + dataAliasController, + bulkDataAliasController, + dataAliasNestedController, + dataAliasExportController, }; diff --git a/packages/nocodb/src/lib/controllers/dataApis/oldDataApis.ts b/packages/nocodb/src/lib/controllers/dataApis/oldDataController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/oldDataApis.ts rename to packages/nocodb/src/lib/controllers/dataApis/oldDataController.ts diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index 390d251bad..b7f89d7ff0 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -1,46 +1,46 @@ import { Tele } from 'nc-help'; -import orgLicenseApis from '../../controllers/orgLicenseController'; -import orgTokenApis from '../../controllers/orgTokenController'; -import orgUserApis from '../../controllers/orgUserController'; -import projectApis from '../../controllers/projectController'; -import baseApis from '../../controllers/baseController'; -import tableApis from '../../controllers/tableController'; -import columnApis from '../../controllers/columnController'; +import orgLicenseController from '../../controllers/orgLicenseController'; +import orgTokenController from '../../controllers/orgTokenController'; +import orgUserController from '../../controllers/orgUserController'; +import projectController from '../../controllers/projectController'; +import baseController from '../../controllers/baseController'; +import tableController from '../../controllers/tableController'; +import columnController from '../../controllers/columnController'; import { Router } from 'express'; -import sortApis from '../../controllers/sortController'; -import filterApis from '../../controllers/filterController'; -import viewColumnApis from '../../controllers/viewColumnController'; -import gridViewApis from '../../controllers/gridViewController'; -import viewApis from '../../controllers/viewController'; -import galleryViewApis from '../../controllers/galleryViewController'; -import formViewApis from '../../controllers/formViewController'; -import formViewColumnApis from '../../controllers/formViewColumnController'; -import attachmentApis from '../../controllers/attachmentController'; -import exportApis from '../../controllers/exportController'; -import auditApis from '../../controllers/auditController'; -import hookApis from '../../controllers/hookController'; -import pluginApis from '../../controllers/pluginController'; -import gridViewColumnApis from '../../controllers/gridViewColumnController'; -import kanbanViewApis from '../../controllers/kanbanViewController'; +import sortController from '../../controllers/sortController'; +import filterController from '../../controllers/filterController'; +import viewColumnController from '../../controllers/viewColumnController'; +import gridViewController from '../../controllers/gridViewController'; +import viewController from '../../controllers/viewController'; +import galleryViewController from '../../controllers/galleryViewController'; +import formViewController from '../../controllers/formViewController'; +import formViewColumnController from '../../controllers/formViewColumnController'; +import attachmentController from '../../controllers/attachmentController'; +import exportController from '../../controllers/exportController'; +import auditController from '../../controllers/auditController'; +import hookController from '../../controllers/hookController'; +import pluginController from '../../controllers/pluginController'; +import gridViewColumnController from '../../controllers/gridViewColumnController'; +import kanbanViewController from '../../controllers/kanbanViewController'; import { userController } from '../../controllers/userController'; // import extractProjectIdAndAuthenticate from './helpers/extractProjectIdAndAuthenticate'; -import utilApis from '../../controllers/utilController'; -import projectUserApis from '../../controllers/projectUserController'; -import sharedBaseApis from '../../controllers/sharedBaseController'; +import utilController from '../../controllers/utilController'; +import projectUserController from '../../controllers/projectUserController'; +import sharedBaseController from '../../controllers/sharedBaseController'; import { initStrategies } from '../../controllers/userController/initStrategies'; -import modelVisibilityApis from '../../controllers/modelVisibilityController'; -import metaDiffApis from '../../controllers/metaDiffController'; -import cacheApis from '../../controllers/cacheController'; -import apiTokenApis from '../../controllers/apiTokenController'; -import hookFilterApis from '../../controllers/hookFilterController'; -import testApis from '../../controllers/testController'; +import modelVisibilityController from '../../controllers/modelVisibilityController'; +import metaDiffController from '../../controllers/metaDiffController'; +import cacheController from '../../controllers/cacheController'; +import apiTokenController from '../../controllers/apiTokenController'; +import hookFilterController from '../../controllers/hookFilterController'; +import testController from '../../controllers/testController'; import { - bulkDataAliasApis, - dataAliasApis, - dataAliasExportApis, - dataAliasNestedApis, - dataApis, - oldDataApis, + bulkDataAliasController, + dataAliasController, + dataAliasExportController, + dataAliasNestedController, + dataController, + oldDataController, } from '../../controllers/dataApis'; import { publicDataController, @@ -51,62 +51,62 @@ import { Server, Socket } from 'socket.io'; import passport from 'passport'; import crypto from 'crypto'; -import swaggerApis from '../../controllers/swaggerController'; -import importApis from '../../controllers/syncController/importApis'; -import syncSourceApis from '../../controllers/syncController'; -import mapViewApis from '../../controllers/mapViewController'; +import swaggerController from '../../controllers/swaggerController'; +import importController from '../../controllers/syncController/importApis'; +import syncSourceController from '../../controllers/syncController'; +import mapViewController from '../../controllers/mapViewController'; const clients: { [id: string]: Socket } = {}; const jobs: { [id: string]: { last_message: any } } = {}; export default function (router: Router, server) { initStrategies(router); - projectApis(router); - baseApis(router); - utilApis(router); + projectController(router); + baseController(router); + utilController(router); if (process.env['PLAYWRIGHT_TEST'] === 'true') { - router.use(testApis); + router.use(testController); } - router.use(columnApis); - router.use(exportApis); - router.use(dataApis); - router.use(bulkDataAliasApis); - router.use(dataAliasApis); - router.use(dataAliasNestedApis); - router.use(dataAliasExportApis); - router.use(oldDataApis); - router.use(sortApis); - router.use(filterApis); - router.use(viewColumnApis); - router.use(gridViewApis); - router.use(formViewColumnApis); + router.use(columnController); + router.use(exportController); + router.use(dataController); + router.use(bulkDataAliasController); + router.use(dataAliasController); + router.use(dataAliasNestedController); + router.use(dataAliasExportController); + router.use(oldDataController); + router.use(sortController); + router.use(filterController); + router.use(viewColumnController); + router.use(gridViewController); + router.use(formViewColumnController); router.use(publicDataController); router.use(publicDataExportController); router.use(publicMetaController); - router.use(gridViewColumnApis); - router.use(tableApis); - router.use(galleryViewApis); - router.use(formViewApis); - router.use(viewApis); - router.use(attachmentApis); - router.use(auditApis); - router.use(hookApis); - router.use(pluginApis); - router.use(projectUserApis); - router.use(orgUserApis); - router.use(orgTokenApis); - router.use(orgLicenseApis); - router.use(sharedBaseApis); - router.use(modelVisibilityApis); - router.use(metaDiffApis); - router.use(cacheApis); - router.use(apiTokenApis); - router.use(hookFilterApis); - router.use(swaggerApis); - router.use(syncSourceApis); - router.use(kanbanViewApis); - router.use(mapViewApis); + router.use(gridViewColumnController); + router.use(tableController); + router.use(galleryViewController); + router.use(formViewController); + router.use(viewController); + router.use(attachmentController); + router.use(auditController); + router.use(hookController); + router.use(pluginController); + router.use(projectUserController); + router.use(orgUserController); + router.use(orgTokenController); + router.use(orgLicenseController); + router.use(sharedBaseController); + router.use(modelVisibilityController); + router.use(metaDiffController); + router.use(cacheController); + router.use(apiTokenController); + router.use(hookFilterController); + router.use(swaggerController); + router.use(syncSourceController); + router.use(kanbanViewController); + router.use(mapViewController); userController(router); @@ -152,7 +152,7 @@ export default function (router: Router, server) { }); }); - importApis(router, io, jobs); + importController(router, io, jobs); } function getHash(str) { From 51781ede2e704f23695f241e520bcced9da325a7 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 10:49:54 +0530 Subject: [PATCH 58/64] refactor: move project and table handler Signed-off-by: Pranav C --- packages/nocodb/src/lib/{meta/api => controllers}/projectApis.ts | 0 .../controllers/{projectController.ts => projectController1.ts} | 0 packages/nocodb/src/lib/{meta/api => controllers}/tableApis.ts | 0 .../lib/controllers/{tableController.ts => tableController1.ts} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename packages/nocodb/src/lib/{meta/api => controllers}/projectApis.ts (100%) rename packages/nocodb/src/lib/controllers/{projectController.ts => projectController1.ts} (100%) rename packages/nocodb/src/lib/{meta/api => controllers}/tableApis.ts (100%) rename packages/nocodb/src/lib/controllers/{tableController.ts => tableController1.ts} (100%) diff --git a/packages/nocodb/src/lib/meta/api/projectApis.ts b/packages/nocodb/src/lib/controllers/projectApis.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/projectApis.ts rename to packages/nocodb/src/lib/controllers/projectApis.ts diff --git a/packages/nocodb/src/lib/controllers/projectController.ts b/packages/nocodb/src/lib/controllers/projectController1.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/projectController.ts rename to packages/nocodb/src/lib/controllers/projectController1.ts diff --git a/packages/nocodb/src/lib/meta/api/tableApis.ts b/packages/nocodb/src/lib/controllers/tableApis.ts similarity index 100% rename from packages/nocodb/src/lib/meta/api/tableApis.ts rename to packages/nocodb/src/lib/controllers/tableApis.ts diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController1.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/tableController.ts rename to packages/nocodb/src/lib/controllers/tableController1.ts From b69bc28fdefb4350e262d1b1f31069ffec944b0d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 10:50:47 +0530 Subject: [PATCH 59/64] refactor: update project and table handler content Signed-off-by: Pranav C --- .../nocodb/src/lib/controllers/projectApis.ts | 169 ++++++++++++++++++ .../nocodb/src/lib/controllers/tableApis.ts | 111 ++++++++++++ 2 files changed, 280 insertions(+) diff --git a/packages/nocodb/src/lib/controllers/projectApis.ts b/packages/nocodb/src/lib/controllers/projectApis.ts index e69de29bb2..18d761efb1 100644 --- a/packages/nocodb/src/lib/controllers/projectApis.ts +++ b/packages/nocodb/src/lib/controllers/projectApis.ts @@ -0,0 +1,169 @@ +import { Request, Response } from 'express'; +import { ProjectType } from 'nocodb-sdk'; +import Project from '../models/Project'; +import { ProjectListType } from 'nocodb-sdk'; +import { packageVersion } from '../utils/packageVersion'; +import { T } from 'nc-help'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import ProjectUser from '../models/ProjectUser'; +import Noco from '../Noco'; +import isDocker from 'is-docker'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import Filter from '../models/Filter'; + +import { projectService } from '../services'; + +// // Project CRUD + +export async function projectGet( + req: Request, + res: Response +) { + const project = await projectService.getProjectWithInfo({ + projectId: req.params.projectId, + }); + + projectService.sanitizeProject(project); + + res.json(project); +} + +export async function projectUpdate( + req: Request, + res: Response +) { + const project = await projectService.projectUpdate({ + projectId: req.params.projectId, + project: req.body, + }); + + res.json(project); +} + +export async function projectList( + req: Request & { user: { id: string; roles: string } }, + res: Response +) { + const projects = await projectService.projectList({ + user: req.user, + query: req.query, + }); + + res.json( + new PagedResponseImpl(projects as ProjectType[], { + count: projects.length, + limit: projects.length, + }) + ); +} + +export async function projectDelete(req: Request, res: Response) { + const deleted = await projectService.projectSoftDelete({ + projectId: req.params.projectId, + }); + + res.json(deleted); +} + +async function projectCreate(req: Request, res) { + const project = await projectService.projectCreate({ + project: req.body, + user: req['user'], + }); + + res.json(project); +} + +export async function projectInfoGet(_req, res) { + res.json({ + Node: process.version, + Arch: process.arch, + Platform: process.platform, + Docker: isDocker(), + RootDB: Noco.getConfig()?.meta?.db?.client, + PackageVersion: packageVersion, + }); +} + +export async function projectCost(req, res) { + let cost = 0; + const project = await Project.getWithInfo(req.params.projectId); + + for (const base of project.bases) { + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + const userCount = await ProjectUser.getUsersCount(req.query); + const recordCount = (await sqlClient.totalRecords())?.data.TotalRecords; + + if (recordCount > 100000) { + // 36,000 or $79/user/month + cost = Math.max(36000, 948 * userCount); + } else if (recordCount > 50000) { + // $36,000 or $50/user/month + cost = Math.max(36000, 600 * userCount); + } else if (recordCount > 10000) { + // $240/user/yr + cost = Math.min(240 * userCount, 36000); + } else if (recordCount > 1000) { + // $120/user/yr + cost = Math.min(120 * userCount, 36000); + } + } + + T.event({ + event: 'a:project:cost', + data: { + cost, + }, + }); + + res.json({ cost }); +} + +export async function hasEmptyOrNullFilters(req, res) { + res.json(await Filter.hasEmptyOrNullFilters(req.params.projectId)); +} + +export default (router) => { + router.get( + '/api/v1/db/meta/projects/:projectId/info', + metaApiMetrics, + ncMetaAclMw(projectInfoGet, 'projectInfoGet') + ); + router.get( + '/api/v1/db/meta/projects/:projectId', + metaApiMetrics, + ncMetaAclMw(projectGet, 'projectGet') + ); + router.patch( + '/api/v1/db/meta/projects/:projectId', + metaApiMetrics, + ncMetaAclMw(projectUpdate, 'projectUpdate') + ); + router.get( + '/api/v1/db/meta/projects/:projectId/cost', + metaApiMetrics, + ncMetaAclMw(projectCost, 'projectCost') + ); + router.delete( + '/api/v1/db/meta/projects/:projectId', + metaApiMetrics, + ncMetaAclMw(projectDelete, 'projectDelete') + ); + router.post( + '/api/v1/db/meta/projects', + metaApiMetrics, + ncMetaAclMw(projectCreate, 'projectCreate') + ); + router.get( + '/api/v1/db/meta/projects', + metaApiMetrics, + ncMetaAclMw(projectList, 'projectList') + ); + router.get( + '/api/v1/db/meta/projects/:projectId/has-empty-or-null-filters', + metaApiMetrics, + ncMetaAclMw(hasEmptyOrNullFilters, 'hasEmptyOrNullFilters') + ); +}; diff --git a/packages/nocodb/src/lib/controllers/tableApis.ts b/packages/nocodb/src/lib/controllers/tableApis.ts index e69de29bb2..c59c8d4a13 100644 --- a/packages/nocodb/src/lib/controllers/tableApis.ts +++ b/packages/nocodb/src/lib/controllers/tableApis.ts @@ -0,0 +1,111 @@ +import { Request, Response, Router } from 'express'; +import { TableListType, TableReqType, TableType } from 'nocodb-sdk'; +import { metaApiMetrics } from '../meta/helpers/apiMetrics'; +import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; +import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; +import { tableService } from '../services'; + +export async function tableList(req: Request, res: Response) { + res.json( + new PagedResponseImpl( + await tableService.getAccessibleTables({ + projectId: req.params.projectId, + baseId: req.params.baseId, + includeM2M: req.query?.includeM2M === 'true', + roles: (req as any).session?.passport?.user?.roles, + }) + ) + ); +} + +export async function tableCreate(req: Request, res) { + const result = await tableService.tableCreate({ + projectId: req.params.projectId, + baseId: req.params.baseId, + table: req.body, + user: (req as any).session?.passport?.user, + }); + + res.json(result); +} + +export async function tableGet(req: Request, res: Response) { + const table = await tableService.getTableWithAccessibleViews({ + tableId: req.params.tableId, + user: (req as any).session?.passport?.user, + }); + + res.json(table); +} + +export async function tableDelete(req: Request, res: Response) { + const result = await tableService.tableDelete({ + tableId: req.params.tableId, + user: (req as any).session?.passport?.user, + req, + }); + + res.json(result); +} + +export async function tableReorder(req: Request, res: Response) { + res.json( + await tableService.reorderTable({ + tableId: req.params.tableId, + order: req.body.order, + }) + ); +} + +// todo: move to table service +export async function tableUpdate(req: Request, res) { + await tableService.tableUpdate({ + tableId: req.params.tableId, + table: req.body, + projectId: (req as any).ncProjectId, + }); + res.json({ msg: 'success' }); +} + +const router = Router({ mergeParams: true }); +router.get( + '/api/v1/db/meta/projects/:projectId/tables', + metaApiMetrics, + ncMetaAclMw(tableList, 'tableList') +); +router.get( + '/api/v1/db/meta/projects/:projectId/:baseId/tables', + metaApiMetrics, + ncMetaAclMw(tableList, 'tableList') +); +router.post( + '/api/v1/db/meta/projects/:projectId/tables', + metaApiMetrics, + ncMetaAclMw(tableCreate, 'tableCreate') +); +router.post( + '/api/v1/db/meta/projects/:projectId/:baseId/tables', + metaApiMetrics, + ncMetaAclMw(tableCreate, 'tableCreate') +); +router.get( + '/api/v1/db/meta/tables/:tableId', + metaApiMetrics, + ncMetaAclMw(tableGet, 'tableGet') +); +router.patch( + '/api/v1/db/meta/tables/:tableId', + metaApiMetrics, + ncMetaAclMw(tableUpdate, 'tableUpdate') +); +router.delete( + '/api/v1/db/meta/tables/:tableId', + metaApiMetrics, + ncMetaAclMw(tableDelete, 'tableDelete') +); +router.post( + '/api/v1/db/meta/tables/:tableId/reorder', + metaApiMetrics, + ncMetaAclMw(tableReorder, 'tableReorder') +); +export default router; From 7e7ce95f0d80103744d34ba973933437dbf443b9 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 10:51:25 +0530 Subject: [PATCH 60/64] refactor: rename project and table handler name Signed-off-by: Pranav C --- .../src/lib/controllers/{projectApis.ts => projectController.ts} | 0 .../src/lib/controllers/{tableApis.ts => tableController.ts} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename packages/nocodb/src/lib/controllers/{projectApis.ts => projectController.ts} (100%) rename packages/nocodb/src/lib/controllers/{tableApis.ts => tableController.ts} (100%) diff --git a/packages/nocodb/src/lib/controllers/projectApis.ts b/packages/nocodb/src/lib/controllers/projectController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/projectApis.ts rename to packages/nocodb/src/lib/controllers/projectController.ts diff --git a/packages/nocodb/src/lib/controllers/tableApis.ts b/packages/nocodb/src/lib/controllers/tableController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/tableApis.ts rename to packages/nocodb/src/lib/controllers/tableController.ts From eb79f50d22b7d19a1041b03dadfeef28d211c4cc Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 10:52:36 +0530 Subject: [PATCH 61/64] refactor: remove duplicate handler Signed-off-by: Pranav C --- .../src/lib/controllers/projectController1.ts | 169 ------------------ .../src/lib/controllers/tableController1.ts | 111 ------------ 2 files changed, 280 deletions(-) delete mode 100644 packages/nocodb/src/lib/controllers/projectController1.ts delete mode 100644 packages/nocodb/src/lib/controllers/tableController1.ts diff --git a/packages/nocodb/src/lib/controllers/projectController1.ts b/packages/nocodb/src/lib/controllers/projectController1.ts deleted file mode 100644 index 18d761efb1..0000000000 --- a/packages/nocodb/src/lib/controllers/projectController1.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { Request, Response } from 'express'; -import { ProjectType } from 'nocodb-sdk'; -import Project from '../models/Project'; -import { ProjectListType } from 'nocodb-sdk'; -import { packageVersion } from '../utils/packageVersion'; -import { T } from 'nc-help'; -import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; -import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import ProjectUser from '../models/ProjectUser'; -import Noco from '../Noco'; -import isDocker from 'is-docker'; -import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import Filter from '../models/Filter'; - -import { projectService } from '../services'; - -// // Project CRUD - -export async function projectGet( - req: Request, - res: Response -) { - const project = await projectService.getProjectWithInfo({ - projectId: req.params.projectId, - }); - - projectService.sanitizeProject(project); - - res.json(project); -} - -export async function projectUpdate( - req: Request, - res: Response -) { - const project = await projectService.projectUpdate({ - projectId: req.params.projectId, - project: req.body, - }); - - res.json(project); -} - -export async function projectList( - req: Request & { user: { id: string; roles: string } }, - res: Response -) { - const projects = await projectService.projectList({ - user: req.user, - query: req.query, - }); - - res.json( - new PagedResponseImpl(projects as ProjectType[], { - count: projects.length, - limit: projects.length, - }) - ); -} - -export async function projectDelete(req: Request, res: Response) { - const deleted = await projectService.projectSoftDelete({ - projectId: req.params.projectId, - }); - - res.json(deleted); -} - -async function projectCreate(req: Request, res) { - const project = await projectService.projectCreate({ - project: req.body, - user: req['user'], - }); - - res.json(project); -} - -export async function projectInfoGet(_req, res) { - res.json({ - Node: process.version, - Arch: process.arch, - Platform: process.platform, - Docker: isDocker(), - RootDB: Noco.getConfig()?.meta?.db?.client, - PackageVersion: packageVersion, - }); -} - -export async function projectCost(req, res) { - let cost = 0; - const project = await Project.getWithInfo(req.params.projectId); - - for (const base of project.bases) { - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - const userCount = await ProjectUser.getUsersCount(req.query); - const recordCount = (await sqlClient.totalRecords())?.data.TotalRecords; - - if (recordCount > 100000) { - // 36,000 or $79/user/month - cost = Math.max(36000, 948 * userCount); - } else if (recordCount > 50000) { - // $36,000 or $50/user/month - cost = Math.max(36000, 600 * userCount); - } else if (recordCount > 10000) { - // $240/user/yr - cost = Math.min(240 * userCount, 36000); - } else if (recordCount > 1000) { - // $120/user/yr - cost = Math.min(120 * userCount, 36000); - } - } - - T.event({ - event: 'a:project:cost', - data: { - cost, - }, - }); - - res.json({ cost }); -} - -export async function hasEmptyOrNullFilters(req, res) { - res.json(await Filter.hasEmptyOrNullFilters(req.params.projectId)); -} - -export default (router) => { - router.get( - '/api/v1/db/meta/projects/:projectId/info', - metaApiMetrics, - ncMetaAclMw(projectInfoGet, 'projectInfoGet') - ); - router.get( - '/api/v1/db/meta/projects/:projectId', - metaApiMetrics, - ncMetaAclMw(projectGet, 'projectGet') - ); - router.patch( - '/api/v1/db/meta/projects/:projectId', - metaApiMetrics, - ncMetaAclMw(projectUpdate, 'projectUpdate') - ); - router.get( - '/api/v1/db/meta/projects/:projectId/cost', - metaApiMetrics, - ncMetaAclMw(projectCost, 'projectCost') - ); - router.delete( - '/api/v1/db/meta/projects/:projectId', - metaApiMetrics, - ncMetaAclMw(projectDelete, 'projectDelete') - ); - router.post( - '/api/v1/db/meta/projects', - metaApiMetrics, - ncMetaAclMw(projectCreate, 'projectCreate') - ); - router.get( - '/api/v1/db/meta/projects', - metaApiMetrics, - ncMetaAclMw(projectList, 'projectList') - ); - router.get( - '/api/v1/db/meta/projects/:projectId/has-empty-or-null-filters', - metaApiMetrics, - ncMetaAclMw(hasEmptyOrNullFilters, 'hasEmptyOrNullFilters') - ); -}; diff --git a/packages/nocodb/src/lib/controllers/tableController1.ts b/packages/nocodb/src/lib/controllers/tableController1.ts deleted file mode 100644 index c59c8d4a13..0000000000 --- a/packages/nocodb/src/lib/controllers/tableController1.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { Request, Response, Router } from 'express'; -import { TableListType, TableReqType, TableType } from 'nocodb-sdk'; -import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; -import { tableService } from '../services'; - -export async function tableList(req: Request, res: Response) { - res.json( - new PagedResponseImpl( - await tableService.getAccessibleTables({ - projectId: req.params.projectId, - baseId: req.params.baseId, - includeM2M: req.query?.includeM2M === 'true', - roles: (req as any).session?.passport?.user?.roles, - }) - ) - ); -} - -export async function tableCreate(req: Request, res) { - const result = await tableService.tableCreate({ - projectId: req.params.projectId, - baseId: req.params.baseId, - table: req.body, - user: (req as any).session?.passport?.user, - }); - - res.json(result); -} - -export async function tableGet(req: Request, res: Response) { - const table = await tableService.getTableWithAccessibleViews({ - tableId: req.params.tableId, - user: (req as any).session?.passport?.user, - }); - - res.json(table); -} - -export async function tableDelete(req: Request, res: Response) { - const result = await tableService.tableDelete({ - tableId: req.params.tableId, - user: (req as any).session?.passport?.user, - req, - }); - - res.json(result); -} - -export async function tableReorder(req: Request, res: Response) { - res.json( - await tableService.reorderTable({ - tableId: req.params.tableId, - order: req.body.order, - }) - ); -} - -// todo: move to table service -export async function tableUpdate(req: Request, res) { - await tableService.tableUpdate({ - tableId: req.params.tableId, - table: req.body, - projectId: (req as any).ncProjectId, - }); - res.json({ msg: 'success' }); -} - -const router = Router({ mergeParams: true }); -router.get( - '/api/v1/db/meta/projects/:projectId/tables', - metaApiMetrics, - ncMetaAclMw(tableList, 'tableList') -); -router.get( - '/api/v1/db/meta/projects/:projectId/:baseId/tables', - metaApiMetrics, - ncMetaAclMw(tableList, 'tableList') -); -router.post( - '/api/v1/db/meta/projects/:projectId/tables', - metaApiMetrics, - ncMetaAclMw(tableCreate, 'tableCreate') -); -router.post( - '/api/v1/db/meta/projects/:projectId/:baseId/tables', - metaApiMetrics, - ncMetaAclMw(tableCreate, 'tableCreate') -); -router.get( - '/api/v1/db/meta/tables/:tableId', - metaApiMetrics, - ncMetaAclMw(tableGet, 'tableGet') -); -router.patch( - '/api/v1/db/meta/tables/:tableId', - metaApiMetrics, - ncMetaAclMw(tableUpdate, 'tableUpdate') -); -router.delete( - '/api/v1/db/meta/tables/:tableId', - metaApiMetrics, - ncMetaAclMw(tableDelete, 'tableDelete') -); -router.post( - '/api/v1/db/meta/tables/:tableId/reorder', - metaApiMetrics, - ncMetaAclMw(tableReorder, 'tableReorder') -); -export default router; From ced48a282c4114706d82fb62357a634cb7042ae4 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 11:17:53 +0530 Subject: [PATCH 62/64] refactor: move nested data(LTAR) apis logic to service Signed-off-by: Pranav C --- .../dataApis/dataAliasNestedController.ts | 248 +++------------- .../dataService/dataAliasNestedService.ts | 280 ++++++++++++++++++ packages/nocodb/src/lib/services/index.ts | 1 + 3 files changed, 328 insertions(+), 201 deletions(-) create mode 100644 packages/nocodb/src/lib/services/dataService/dataAliasNestedService.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedController.ts b/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedController.ts index 3398bdffa3..9fc5169ee3 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedController.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedController.ts @@ -1,223 +1,79 @@ import { Request, Response, Router } from 'express'; -import Model from '../../models/Model'; -import Base from '../../models/Base'; -import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2' -import { - getColumnByIdOrName, - getViewAndModelFromRequestByAliasOrId, -} from './helpers'; -import { NcError } from '../../meta/helpers/catchError'; import apiMetrics from '../../meta/helpers/apiMetrics'; +import { dataAliasNestedService } from '../../services'; // todo: handle case where the given column is not ltar -export async function mmList(req: Request, res: Response, next) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - - const data = await baseModel.mmList( - { - colId: column.id, - parentId: req.params.rowId, - }, - req.query as any - ); - const count: any = await baseModel.mmListCount({ - colId: column.id, - parentId: req.params.rowId, - }); - +export async function mmList(req: Request, res: Response) { res.json( - new PagedResponseImpl(data, { - count, - ...req.query, + await dataAliasNestedService.mmList({ + query: req.query, + columnName: req.params.columnName, + rowId: req.params.rowId, + projectName: req.params.projectName, + tableName: req.params.tableName, }) ); } -export async function mmExcludedList(req: Request, res: Response, next) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - const column = await getColumnByIdOrName(req.params.columnName, model); - - const data = await baseModel.getMmChildrenExcludedList( - { - colId: column.id, - pid: req.params.rowId, - }, - req.query - ); - - const count = await baseModel.getMmChildrenExcludedListCount( - { - colId: column.id, - pid: req.params.rowId, - }, - req.query - ); - +export async function mmExcludedList(req: Request, res: Response) { res.json( - new PagedResponseImpl(data, { - count, - ...req.query, + await dataAliasNestedService.mmExcludedList({ + query: req.query, + columnName: req.params.columnName, + rowId: req.params.rowId, + projectName: req.params.projectName, + tableName: req.params.tableName, }) ); } -export async function hmExcludedList(req: Request, res: Response, next) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - - const data = await baseModel.getHmChildrenExcludedList( - { - colId: column.id, - pid: req.params.rowId, - }, - req.query - ); - - const count = await baseModel.getHmChildrenExcludedListCount( - { - colId: column.id, - pid: req.params.rowId, - }, - req.query - ); - +export async function hmExcludedList(req: Request, res: Response) { res.json( - new PagedResponseImpl(data, { - count, - ...req.query, + await dataAliasNestedService.hmExcludedList({ + query: req.query, + columnName: req.params.columnName, + rowId: req.params.rowId, + projectName: req.params.projectName, + tableName: req.params.tableName, }) ); } -export async function btExcludedList(req: Request, res: Response, next) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - - const data = await baseModel.getBtChildrenExcludedList( - { - colId: column.id, - cid: req.params.rowId, - }, - req.query - ); - - const count = await baseModel.getBtChildrenExcludedListCount( - { - colId: column.id, - cid: req.params.rowId, - }, - req.query - ); - +export async function btExcludedList(req: Request, res: Response) { res.json( - new PagedResponseImpl(data, { - count, - ...req.query, + await dataAliasNestedService.btExcludedList({ + query: req.query, + columnName: req.params.columnName, + rowId: req.params.rowId, + projectName: req.params.projectName, + tableName: req.params.tableName, }) ); } // todo: handle case where the given column is not ltar -export async function hmList(req: Request, res: Response, next) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - if (!model) return next(new Error('Table not found')); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - - const data = await baseModel.hmList( - { - colId: column.id, - id: req.params.rowId, - }, - req.query - ); - - const count = await baseModel.hmListCount({ - colId: column.id, - id: req.params.rowId, - }); - +export async function hmList(req: Request, res: Response) { res.json( - new PagedResponseImpl(data, { - count, - ...req.query, - } as any) + await dataAliasNestedService.hmList({ + query: req.query, + columnName: req.params.columnName, + rowId: req.params.rowId, + projectName: req.params.projectName, + tableName: req.params.tableName, + }) ); } //@ts-ignore async function relationDataRemove(req, res) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - if (!model) NcError.notFound('Table not found'); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - - await baseModel.removeChild({ - colId: column.id, - childId: req.params.refRowId, + await dataAliasNestedService.relationDataRemove({ + columnName: req.params.columnName, rowId: req.params.rowId, + projectName: req.params.projectName, + tableName: req.params.tableName, cookie: req, + refRowId: req.params.refRowId, }); res.json({ msg: 'success' }); @@ -226,23 +82,13 @@ async function relationDataRemove(req, res) { //@ts-ignore // todo: Give proper error message when reference row is already related and handle duplicate ref row id in hm async function relationDataAdd(req, res) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - if (!model) NcError.notFound('Table not found'); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const column = await getColumnByIdOrName(req.params.columnName, model); - await baseModel.addChild({ - colId: column.id, - childId: req.params.refRowId, + await dataAliasNestedService.relationDataAdd({ + columnName: req.params.columnName, rowId: req.params.rowId, + projectName: req.params.projectName, + tableName: req.params.tableName, cookie: req, + refRowId: req.params.refRowId, }); res.json({ msg: 'success' }); diff --git a/packages/nocodb/src/lib/services/dataService/dataAliasNestedService.ts b/packages/nocodb/src/lib/services/dataService/dataAliasNestedService.ts new file mode 100644 index 0000000000..20959d15fa --- /dev/null +++ b/packages/nocodb/src/lib/services/dataService/dataAliasNestedService.ts @@ -0,0 +1,280 @@ +import { PagedResponseImpl } from '../../meta/helpers/PagedResponse'; +import { Base, Model } from '../../models'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import { + getColumnByIdOrName, + getViewAndModelByAliasOrId, + PathParams, +} from './helpers'; +import { NcError } from '../../meta/helpers/catchError'; + +// todo: handle case where the given column is not ltar +export async function mmList( + param: PathParams & { + query: any; + columnName: string; + rowId: string; + } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(param.columnName, model); + + const data = await baseModel.mmList( + { + colId: column.id, + parentId: param.rowId, + }, + param.query as any + ); + const count: any = await baseModel.mmListCount({ + colId: column.id, + parentId: param.rowId, + }); + + return new PagedResponseImpl(data, { + count, + ...param.query, + }); +} + +export async function mmExcludedList( + param: PathParams & { + query: any; + columnName: string; + rowId: string; + } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + const column = await getColumnByIdOrName(param.columnName, model); + + const data = await baseModel.getMmChildrenExcludedList( + { + colId: column.id, + pid: param.rowId, + }, + param.query + ); + + const count = await baseModel.getMmChildrenExcludedListCount( + { + colId: column.id, + pid: param.rowId, + }, + param.query + ); + + return new PagedResponseImpl(data, { + count, + ...param.query, + }); +} + +export async function hmExcludedList( + param: PathParams & { + query: any; + columnName: string; + rowId: string; + } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(param.columnName, model); + + const data = await baseModel.getHmChildrenExcludedList( + { + colId: column.id, + pid: param.rowId, + }, + param.query + ); + + const count = await baseModel.getHmChildrenExcludedListCount( + { + colId: column.id, + pid: param.rowId, + }, + param.query + ); + + return new PagedResponseImpl(data, { + count, + ...param.query, + }); +} + +export async function btExcludedList( + param: PathParams & { + query: any; + columnName: string; + rowId: string; + } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(param.columnName, model); + + const data = await baseModel.getBtChildrenExcludedList( + { + colId: column.id, + cid: param.rowId, + }, + param.query + ); + + const count = await baseModel.getBtChildrenExcludedListCount( + { + colId: column.id, + cid: param.rowId, + }, + param.query + ); + + return new PagedResponseImpl(data, { + count, + ...param.query, + }); +} + +// todo: handle case where the given column is not ltar +export async function hmList( + param: PathParams & { + query: any; + columnName: string; + rowId: string; + } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(param.columnName, model); + + const data = await baseModel.hmList( + { + colId: column.id, + id: param.rowId, + }, + param.query + ); + + const count = await baseModel.hmListCount({ + colId: column.id, + id: param.rowId, + }); + + return new PagedResponseImpl(data, { + count, + ...param.query, + } as any); +} + +//@ts-ignore +export async function relationDataRemove( + param: PathParams & { + columnName: string; + rowId: string; + refRowId: string; + cookie: any; + } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(param.columnName, model); + + await baseModel.removeChild({ + colId: column.id, + childId: param.refRowId, + rowId: param.rowId, + cookie: param.cookie, + }); + + return true; +} + +//@ts-ignore +// todo: Give proper error message when reference row is already related and handle duplicate ref row id in hm +export async function relationDataAdd( + param: PathParams & { + columnName: string; + rowId: string; + refRowId: string; + cookie: any; + } +) { + const { model, view } = await getViewAndModelByAliasOrId(param); + if (!model) NcError.notFound('Table not found'); + + const base = await Base.get(model.base_id); + + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: view?.id, + dbDriver: NcConnectionMgrv2.get(base), + }); + + const column = await getColumnByIdOrName(param.columnName, model); + await baseModel.addChild({ + colId: column.id, + childId: param.refRowId, + rowId: param.rowId, + cookie: param.cookie, + }); + + return true; +} diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index f99398753c..c58049aa62 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -27,6 +27,7 @@ export * as attachmentService from './attachmentService'; export * as hookFilterService from './hookFilterService'; export * as dataService from './dataService'; export * as bulkDataService from './dataService/bulkData'; +export * as dataAliasNestedService from './dataService/dataAliasNestedService'; export * as cacheService from './cacheService'; export * as auditService from './auditService'; export * as swaggerService from './swaggerService'; From 842819c836078375de3ecc97639d2c6337710e2e Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 11:23:15 +0530 Subject: [PATCH 63/64] refactor: rename folders Signed-off-by: Pranav C --- .../{dataApis => dataControllers}/bulkDataAliasController.ts | 0 .../{dataApis => dataControllers}/dataAliasController.ts | 0 .../dataAliasExportController.ts | 0 .../dataAliasNestedController.ts | 0 .../{dataApis => dataControllers}/dataController.ts | 0 .../lib/controllers/{dataApis => dataControllers}/helpers.ts | 0 .../lib/controllers/{dataApis => dataControllers}/index.ts | 0 .../{dataApis => dataControllers}/oldDataController.ts | 0 packages/nocodb/src/lib/controllers/exportController.ts | 2 +- .../lib/controllers/{public => publicControllers}/index.ts | 0 .../{public => publicControllers}/publicDataController.ts | 0 .../publicDataExportController.ts | 0 .../{public => publicControllers}/publicMetaController.ts | 0 packages/nocodb/src/lib/meta/api/index.ts | 4 ++-- 14 files changed, 3 insertions(+), 3 deletions(-) rename packages/nocodb/src/lib/controllers/{dataApis => dataControllers}/bulkDataAliasController.ts (100%) rename packages/nocodb/src/lib/controllers/{dataApis => dataControllers}/dataAliasController.ts (100%) rename packages/nocodb/src/lib/controllers/{dataApis => dataControllers}/dataAliasExportController.ts (100%) rename packages/nocodb/src/lib/controllers/{dataApis => dataControllers}/dataAliasNestedController.ts (100%) rename packages/nocodb/src/lib/controllers/{dataApis => dataControllers}/dataController.ts (100%) rename packages/nocodb/src/lib/controllers/{dataApis => dataControllers}/helpers.ts (100%) rename packages/nocodb/src/lib/controllers/{dataApis => dataControllers}/index.ts (100%) rename packages/nocodb/src/lib/controllers/{dataApis => dataControllers}/oldDataController.ts (100%) rename packages/nocodb/src/lib/controllers/{public => publicControllers}/index.ts (100%) rename packages/nocodb/src/lib/controllers/{public => publicControllers}/publicDataController.ts (100%) rename packages/nocodb/src/lib/controllers/{public => publicControllers}/publicDataExportController.ts (100%) rename packages/nocodb/src/lib/controllers/{public => publicControllers}/publicMetaController.ts (100%) diff --git a/packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasController.ts b/packages/nocodb/src/lib/controllers/dataControllers/bulkDataAliasController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/bulkDataAliasController.ts rename to packages/nocodb/src/lib/controllers/dataControllers/bulkDataAliasController.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasController.ts b/packages/nocodb/src/lib/controllers/dataControllers/dataAliasController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/dataAliasController.ts rename to packages/nocodb/src/lib/controllers/dataControllers/dataAliasController.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasExportController.ts b/packages/nocodb/src/lib/controllers/dataControllers/dataAliasExportController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/dataAliasExportController.ts rename to packages/nocodb/src/lib/controllers/dataControllers/dataAliasExportController.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedController.ts b/packages/nocodb/src/lib/controllers/dataControllers/dataAliasNestedController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/dataAliasNestedController.ts rename to packages/nocodb/src/lib/controllers/dataControllers/dataAliasNestedController.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/dataController.ts b/packages/nocodb/src/lib/controllers/dataControllers/dataController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/dataController.ts rename to packages/nocodb/src/lib/controllers/dataControllers/dataController.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts b/packages/nocodb/src/lib/controllers/dataControllers/helpers.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/helpers.ts rename to packages/nocodb/src/lib/controllers/dataControllers/helpers.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/index.ts b/packages/nocodb/src/lib/controllers/dataControllers/index.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/index.ts rename to packages/nocodb/src/lib/controllers/dataControllers/index.ts diff --git a/packages/nocodb/src/lib/controllers/dataApis/oldDataController.ts b/packages/nocodb/src/lib/controllers/dataControllers/oldDataController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/dataApis/oldDataController.ts rename to packages/nocodb/src/lib/controllers/dataControllers/oldDataController.ts diff --git a/packages/nocodb/src/lib/controllers/exportController.ts b/packages/nocodb/src/lib/controllers/exportController.ts index 1c043acff1..38ae156cb9 100644 --- a/packages/nocodb/src/lib/controllers/exportController.ts +++ b/packages/nocodb/src/lib/controllers/exportController.ts @@ -1,7 +1,7 @@ import { Request, Response, Router } from 'express'; import View from '../models/View'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { extractCsvData } from './dataApis/helpers'; +import { extractCsvData } from './dataControllers/helpers'; async function exportCsv(req: Request, res: Response) { const view = await View.get(req.params.viewId); diff --git a/packages/nocodb/src/lib/controllers/public/index.ts b/packages/nocodb/src/lib/controllers/publicControllers/index.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/public/index.ts rename to packages/nocodb/src/lib/controllers/publicControllers/index.ts diff --git a/packages/nocodb/src/lib/controllers/public/publicDataController.ts b/packages/nocodb/src/lib/controllers/publicControllers/publicDataController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/public/publicDataController.ts rename to packages/nocodb/src/lib/controllers/publicControllers/publicDataController.ts diff --git a/packages/nocodb/src/lib/controllers/public/publicDataExportController.ts b/packages/nocodb/src/lib/controllers/publicControllers/publicDataExportController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/public/publicDataExportController.ts rename to packages/nocodb/src/lib/controllers/publicControllers/publicDataExportController.ts diff --git a/packages/nocodb/src/lib/controllers/public/publicMetaController.ts b/packages/nocodb/src/lib/controllers/publicControllers/publicMetaController.ts similarity index 100% rename from packages/nocodb/src/lib/controllers/public/publicMetaController.ts rename to packages/nocodb/src/lib/controllers/publicControllers/publicMetaController.ts diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index b7f89d7ff0..6b9176188a 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -41,12 +41,12 @@ import { dataAliasNestedController, dataController, oldDataController, -} from '../../controllers/dataApis'; +} from '../../controllers/dataControllers'; import { publicDataController, publicDataExportController, publicMetaController, -} from '../../controllers/public'; +} from '../../controllers/publicControllers'; import { Server, Socket } from 'socket.io'; import passport from 'passport'; From c0ad5b1d1ade5e3b41ecd3fd5d842b3f983b7ade Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 4 Mar 2023 14:40:38 +0530 Subject: [PATCH 64/64] refactor: suggested changes Signed-off-by: Pranav C --- .../nocodb/src/lib/controllers/auditController.ts | 8 ++------ .../dataControllers/bulkDataAliasController.ts | 5 +---- .../dataControllers/dataAliasController.ts | 2 +- .../dataControllers/dataAliasExportController.ts | 2 +- .../src/lib/controllers/userController/userApis.ts | 14 +++++++------- packages/nocodb/src/lib/meta/api/index.ts | 8 ++++---- .../nocodb/src/lib/services/attachmentService.ts | 5 ----- packages/nocodb/src/lib/services/auditService.ts | 14 +++++++------- .../nocodb/src/lib/services/dataService/export.ts | 6 +----- .../nocodb/src/lib/services/projectUserService.ts | 5 ++--- .../src/lib/services/syncService/helpers/job.ts | 6 ++---- packages/nocodb/src/lib/services/tableService.ts | 3 +-- 12 files changed, 29 insertions(+), 49 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/auditController.ts b/packages/nocodb/src/lib/controllers/auditController.ts index 366fc70529..73ab7dafa4 100644 --- a/packages/nocodb/src/lib/controllers/auditController.ts +++ b/packages/nocodb/src/lib/controllers/auditController.ts @@ -9,9 +9,7 @@ export async function commentRow(req: Request, res) { await auditService.commentRow({ rowId: req.params.rowId, user: (req as any).user, - body: { - ...req.body, - }, + body: req.body, }) ); } @@ -20,9 +18,7 @@ export async function auditRowUpdate(req: Request, res) { res.json( await auditService.auditRowUpdate({ rowId: req.params.rowId, - body: { - ...req.body, - }, + body: req.body, }) ); } diff --git a/packages/nocodb/src/lib/controllers/dataControllers/bulkDataAliasController.ts b/packages/nocodb/src/lib/controllers/dataControllers/bulkDataAliasController.ts index 968ab16165..7aa016ef2e 100644 --- a/packages/nocodb/src/lib/controllers/dataControllers/bulkDataAliasController.ts +++ b/packages/nocodb/src/lib/controllers/dataControllers/bulkDataAliasController.ts @@ -1,10 +1,8 @@ import { Request, Response, Router } from 'express'; -import { bulkDataService } from '../../services' +import { bulkDataService } from '../../services'; import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; import apiMetrics from '../../meta/helpers/apiMetrics'; - - async function bulkDataInsert(req: Request, res: Response) { res.json( await bulkDataService.bulkDataInsert({ @@ -63,7 +61,6 @@ async function bulkDataDeleteAll(req: Request, res: Response) { ); } - const router = Router({ mergeParams: true }); router.post( diff --git a/packages/nocodb/src/lib/controllers/dataControllers/dataAliasController.ts b/packages/nocodb/src/lib/controllers/dataControllers/dataAliasController.ts index cfa35fafe6..b5eda23c74 100644 --- a/packages/nocodb/src/lib/controllers/dataControllers/dataAliasController.ts +++ b/packages/nocodb/src/lib/controllers/dataControllers/dataAliasController.ts @@ -1,5 +1,5 @@ import { Request, Response, Router } from 'express'; -import { dataService } from '../../services' +import { dataService } from '../../services'; import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; import apiMetrics from '../../meta/helpers/apiMetrics'; import { parseHrtimeToSeconds } from '../../meta/api/helpers'; diff --git a/packages/nocodb/src/lib/controllers/dataControllers/dataAliasExportController.ts b/packages/nocodb/src/lib/controllers/dataControllers/dataAliasExportController.ts index d67f58f028..0b85b87801 100644 --- a/packages/nocodb/src/lib/controllers/dataControllers/dataAliasExportController.ts +++ b/packages/nocodb/src/lib/controllers/dataControllers/dataAliasExportController.ts @@ -7,7 +7,7 @@ import { extractCsvData, extractXlsxData, } from '../../services/dataService/helpers'; -import { getViewAndModelFromRequestByAliasOrId } from './helpers' +import { getViewAndModelFromRequestByAliasOrId } from './helpers'; async function excelDataExport(req: Request, res: Response) { const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); diff --git a/packages/nocodb/src/lib/controllers/userController/userApis.ts b/packages/nocodb/src/lib/controllers/userController/userApis.ts index 574862e25a..70d17b40ef 100644 --- a/packages/nocodb/src/lib/controllers/userController/userApis.ts +++ b/packages/nocodb/src/lib/controllers/userController/userApis.ts @@ -137,13 +137,13 @@ export async function signup(req: Request, res: Response) { } async function successfulSignIn({ - user, - err, - info, - req, - res, - auditDescription, - }) { + user, + err, + info, + req, + res, + auditDescription, +}) { try { if (!user || !user.email) { if (err) { diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index 6b9176188a..a785e5ad77 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -1,4 +1,4 @@ -import { Tele } from 'nc-help'; +import { T } from 'nc-help'; import orgLicenseController from '../../controllers/orgLicenseController'; import orgTokenController from '../../controllers/orgTokenController'; import orgUserController from '../../controllers/orgUserController'; @@ -133,15 +133,15 @@ export default function (router: Router, server) { }).on('connection', (socket) => { clients[socket.id] = socket; const id = getHash( - (process.env.NC_SERVER_UUID || Tele.id) + + (process.env.NC_SERVER_UUID || T.id) + (socket?.handshake as any)?.user?.id ); socket.on('page', (args) => { - Tele.page({ ...args, id }); + T.page({ ...args, id }); }); socket.on('event', (args) => { - Tele.event({ ...args, id }); + T.event({ ...args, id }); }); socket.on('subscribe', (room) => { if (room in jobs) { diff --git a/packages/nocodb/src/lib/services/attachmentService.ts b/packages/nocodb/src/lib/services/attachmentService.ts index 1d1fc25f02..d65134e75b 100644 --- a/packages/nocodb/src/lib/services/attachmentService.ts +++ b/packages/nocodb/src/lib/services/attachmentService.ts @@ -1,8 +1,3 @@ - - - -// @ts-ignore -import { Request, Response, Router } from 'express'; import { nanoid } from 'nanoid'; import path from 'path'; import slash from 'slash'; diff --git a/packages/nocodb/src/lib/services/auditService.ts b/packages/nocodb/src/lib/services/auditService.ts index f5f2724ba3..886934cc84 100644 --- a/packages/nocodb/src/lib/services/auditService.ts +++ b/packages/nocodb/src/lib/services/auditService.ts @@ -1,10 +1,10 @@ -import { - AuditRowUpdatePayloadType, - CommentRowPayloadType, -} from 'nocodb-sdk/build/main/lib/CustomAPI'; import { validatePayload } from '../meta/api/helpers'; import Audit from '../models/Audit'; -import { AuditOperationSubTypes, AuditOperationTypes } from 'nocodb-sdk'; +import { + AuditOperationSubTypes, + AuditOperationTypes, + AuditRowUpdateReqType, +} from 'nocodb-sdk'; import Model from '../models/Model'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; @@ -12,7 +12,7 @@ import DOMPurify from 'isomorphic-dompurify'; export async function commentRow(param: { rowId: string; - body: CommentRowPayloadType; + body: AuditRowUpdateReqType; user: any; }) { await validatePayload( @@ -29,7 +29,7 @@ export async function commentRow(param: { export async function auditRowUpdate(param: { rowId: string; - body: AuditRowUpdatePayloadType; + body: AuditRowUpdateReqType; }) { await validatePayload( 'swagger.json#/components/schemas/AuditRowUpdateReq', diff --git a/packages/nocodb/src/lib/services/dataService/export.ts b/packages/nocodb/src/lib/services/dataService/export.ts index 53afe1ff85..fa222f61ce 100644 --- a/packages/nocodb/src/lib/services/dataService/export.ts +++ b/packages/nocodb/src/lib/services/dataService/export.ts @@ -1,10 +1,6 @@ -import { - getViewAndModelByAliasOrId, - PathParams, -} from './helpers'; +import { getViewAndModelByAliasOrId, PathParams } from './helpers'; import { View } from '../../models'; - // Todo: bring logic from controller export async function excelDataExport( param: PathParams & { diff --git a/packages/nocodb/src/lib/services/projectUserService.ts b/packages/nocodb/src/lib/services/projectUserService.ts index 12e6a92735..675ba744a6 100644 --- a/packages/nocodb/src/lib/services/projectUserService.ts +++ b/packages/nocodb/src/lib/services/projectUserService.ts @@ -292,9 +292,8 @@ export async function sendInviteEmail( req: any ): Promise { try { - const template = ( - await import('./userService/ui/emailTemplates/invite') - ).default; + const template = (await import('./userService/ui/emailTemplates/invite')) + .default; const emailAdapter = await NcPluginMgrv2.emailAdapter(); diff --git a/packages/nocodb/src/lib/services/syncService/helpers/job.ts b/packages/nocodb/src/lib/services/syncService/helpers/job.ts index b956a5353c..8d6bc80ed7 100644 --- a/packages/nocodb/src/lib/services/syncService/helpers/job.ts +++ b/packages/nocodb/src/lib/services/syncService/helpers/job.ts @@ -679,8 +679,6 @@ export default async ( ); } - // debug - // console.log(JSON.stringify(tables, null, 2)); return tables; } @@ -1413,7 +1411,7 @@ export default async ( case UITypes.DateTime: case UITypes.CreateTime: case UITypes.LastModifiedTime: - rec[key] = dayjs(value).utc().format('YYYY-MM-DD HH:mm'); + rec[key] = dayjs(value).format('YYYY-MM-DD HH:mm'); break; case UITypes.Date: @@ -1422,7 +1420,7 @@ export default async ( rec[key] = null; logBasic(`:: Invalid date ${value}`); } else { - rec[key] = dayjs(value).utc().format('YYYY-MM-DD'); + rec[key] = dayjs(value).format('YYYY-MM-DD'); } break; diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts index eb94a9c708..747f2d57b8 100644 --- a/packages/nocodb/src/lib/services/tableService.ts +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -51,7 +51,7 @@ export async function tableUpdate(param: { if ('meta' in param.table) { await Model.updateMeta(param.tableId, param.table.meta); - return { msg: 'success' } + return true; } if (!param.table.table_name) { @@ -201,7 +201,6 @@ export async function tableDelete(param: { return table.delete(); } - export async function getTableWithAccessibleViews(param: { tableId: string; user: User;