From f17ab9819637ddcb19dbb736676708258cda5464 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 27 Oct 2023 18:26:05 +0530 Subject: [PATCH 001/167] fix: include all columns in webhook payload when inserting from shared form Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 30 ++++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index b2d2ae7b2f..ffca7189a0 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2612,30 +2612,30 @@ class BaseModelSqlv2 { })) as any ).rows[0].id; } - response = await this.readByPk( - id, - false, - {}, - { ignoreView: true, getHiddenColumn: true }, - ); + // response = await this.readByPk( + // id, + // false, + // {}, + // { ignoreView: true, getHiddenColumn: true }, + // ); } else { response = data; } } else if (ai) { - response = await this.readByPk( - Array.isArray(response) + id = Array.isArray(response) ? response?.[0]?.[ai.title] - : response?.[ai.title], - ); + : response?.[ai.title]; } - response = Array.isArray(response) ? response[0] : response; - if (response) - rowId = - response[this.model.primaryKey.title] || - response[this.model.primaryKey.column_name]; + rowId = id; await Promise.all(postInsertOps.map((f) => f())); + const response = this.readByPk(rowId, + false, + {}, + { ignoreView: true, getHiddenColumn: true }, + ); + await this.afterInsert(response, this.dbDriver, cookie); return response; From dd6951938370ef6731347bf6851f378904289c7c Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 27 Oct 2023 18:43:01 +0530 Subject: [PATCH 002/167] fix: use nested insert for insert api Signed-off-by: Pranav C --- packages/nocodb/src/services/datas.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/services/datas.service.ts b/packages/nocodb/src/services/datas.service.ts index f9fb1b7539..11c46c690c 100644 --- a/packages/nocodb/src/services/datas.service.ts +++ b/packages/nocodb/src/services/datas.service.ts @@ -78,7 +78,7 @@ export class DatasService { dbDriver: await NcConnectionMgrv2.get(source), }); - return await baseModel.insert(param.body, null, param.cookie); + return await baseModel.nestedInsert(param.body, null, param.cookie); } async dataUpdate( From faaaff3a9082c378167bd65b06928569c7f49135 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 27 Oct 2023 23:31:14 +0530 Subject: [PATCH 003/167] feat: use nested insert for row insert Signed-off-by: Pranav C --- packages/nc-gui/components/smartsheet/Row.vue | 12 +++++------ .../smartsheet/expanded-form/index.vue | 6 +++--- .../components/smartsheet/grid/Table.vue | 2 ++ packages/nc-gui/composables/useData.ts | 2 +- .../composables/useExpandedFormStore.ts | 2 +- packages/nocodb/src/db/BaseModelSqlv2.ts | 21 +++++++++---------- 6 files changed, 23 insertions(+), 22 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/Row.vue b/packages/nc-gui/components/smartsheet/Row.vue index 39f313c052..5a305bdc02 100644 --- a/packages/nc-gui/components/smartsheet/Row.vue +++ b/packages/nc-gui/components/smartsheet/Row.vue @@ -26,12 +26,12 @@ const { isNew, state, syncLTARRefs, clearLTARCell, addLTARRef } = useProvideSmar // on changing isNew(new record insert) status sync LTAR cell values watch(isNew, async (nextVal, prevVal) => { - if (prevVal && !nextVal) { - await syncLTARRefs(currentRow.value.row) - // update row values without invoking api - currentRow.value.row = { ...currentRow.value.row, ...state.value } - currentRow.value.oldRow = { ...currentRow.value.row, ...state.value } - } + // if (prevVal && !nextVal) { + // await syncLTARRefs(currentRow.value.row) + // // update row values without invoking api + // currentRow.value.row = { ...currentRow.value.row, ...state.value } + // currentRow.value.oldRow = { ...currentRow.value.row, ...state.value } + // } }) const reloadViewDataTrigger = inject(ReloadViewDataHookInj)! diff --git a/packages/nc-gui/components/smartsheet/expanded-form/index.vue b/packages/nc-gui/components/smartsheet/expanded-form/index.vue index ae8c2dd4c5..0cc191d6a3 100644 --- a/packages/nc-gui/components/smartsheet/expanded-form/index.vue +++ b/packages/nc-gui/components/smartsheet/expanded-form/index.vue @@ -185,7 +185,7 @@ const onDuplicateRow = () => { const save = async () => { if (isNew.value) { const data = await _save(rowState.value) - await syncLTARRefs(data) + // await syncLTARRefs(data) reloadTrigger?.trigger() } else { let kanbanClbk @@ -341,7 +341,7 @@ useActiveKeyupListener( if (isNew.value) { const data = await _save(rowState.value) - await syncLTARRefs(data) + // await syncLTARRefs(data) reloadHook?.trigger(null) } else { await save() @@ -376,7 +376,7 @@ useActiveKeyupListener( cancelText: t('labels.discard'), onOk: async () => { const data = await _save(rowState.value) - await syncLTARRefs(data) + // await syncLTARRefs(data) reloadHook?.trigger(null) addNewRow() }, diff --git a/packages/nc-gui/components/smartsheet/grid/Table.vue b/packages/nc-gui/components/smartsheet/grid/Table.vue index e9aeb6e1b2..427232fb22 100644 --- a/packages/nc-gui/components/smartsheet/grid/Table.vue +++ b/packages/nc-gui/components/smartsheet/grid/Table.vue @@ -1454,6 +1454,8 @@ const loaderText = computed(() => { :style="{ height: rowHeight ? `${rowHeight * 1.8}rem` : `1.8rem` }" :data-testid="`grid-row-${rowIndex}`" > + {{ +row.row}} f())); - const response = this.readByPk(rowId, + response = this.readByPk( + rowId, false, {}, { ignoreView: true, getHiddenColumn: true }, - ); + ); await this.afterInsert(response, this.dbDriver, cookie); From ba16b2989a43686f9751be9b3c093a82abd2c24d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 28 Oct 2023 16:30:12 +0530 Subject: [PATCH 004/167] fix: if invalid json then skip Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index ca4e4b1f41..602018b6e8 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2517,11 +2517,15 @@ class BaseModelSqlv2 { await col.getColOptions(); // parse data if it's JSON string - const nestedData = - typeof data[col.title] === 'string' - ? JSON.parse(data[col.title]) - : data[col.title]; - + let nestedData; + try { + nestedData = + typeof data[col.title] === 'string' + ? JSON.parse(data[col.title]) + : data[col.title]; + } catch { + continue + } switch (colOptions.type) { case RelationTypes.BELONGS_TO: { @@ -2628,7 +2632,7 @@ class BaseModelSqlv2 { await Promise.all(postInsertOps.map((f) => f())); - response = this.readByPk( + response = this.readByPk( rowId, false, {}, From a1f489a712278077d028a241800eb1a8157a59e4 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 28 Oct 2023 21:07:32 +0530 Subject: [PATCH 005/167] fix: missing await Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 602018b6e8..00e7c0ff12 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2632,7 +2632,7 @@ class BaseModelSqlv2 { await Promise.all(postInsertOps.map((f) => f())); - response = this.readByPk( + response = await this.readByPk( rowId, false, {}, From 4d7f7d224bf68e1c7a7f7772118f5e3b15f3ed25 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 28 Oct 2023 21:22:30 +0530 Subject: [PATCH 006/167] fix: cleanup Signed-off-by: Pranav C --- packages/nc-gui/components/smartsheet/grid/Table.vue | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/grid/Table.vue b/packages/nc-gui/components/smartsheet/grid/Table.vue index 427232fb22..e9aeb6e1b2 100644 --- a/packages/nc-gui/components/smartsheet/grid/Table.vue +++ b/packages/nc-gui/components/smartsheet/grid/Table.vue @@ -1454,8 +1454,6 @@ const loaderText = computed(() => { :style="{ height: rowHeight ? `${rowHeight * 1.8}rem` : `1.8rem` }" :data-testid="`grid-row-${rowIndex}`" > - {{ -row.row}} Date: Mon, 30 Oct 2023 18:59:32 +0530 Subject: [PATCH 007/167] fix: show links count based on row state for new row Signed-off-by: Pranav C --- packages/nc-gui/components/virtual-cell/Links.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nc-gui/components/virtual-cell/Links.vue b/packages/nc-gui/components/virtual-cell/Links.vue index 51179a4cd1..78668353aa 100644 --- a/packages/nc-gui/components/virtual-cell/Links.vue +++ b/packages/nc-gui/components/virtual-cell/Links.vue @@ -47,7 +47,7 @@ const relatedTableDisplayColumn = computed( loadRelatedTableMeta() const textVal = computed(() => { - if (isForm?.value) { + if (isForm?.value || isNew.value) { return state.value?.[colTitle.value]?.length ? `${+state.value?.[colTitle.value]?.length} ${t('msg.recordsLinked')}` : t('msg.noRecordsLinked') From ac930359999b2fba24d185b894c3a3635e9509c8 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 31 Oct 2023 00:35:10 +0530 Subject: [PATCH 008/167] fix: avoid duplicate api call in form view Signed-off-by: Pranav C --- packages/nc-gui/components/smartsheet/Form.vue | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/Form.vue b/packages/nc-gui/components/smartsheet/Form.vue index 670b7c5c9e..fd8de40185 100644 --- a/packages/nc-gui/components/smartsheet/Form.vue +++ b/packages/nc-gui/components/smartsheet/Form.vue @@ -67,7 +67,7 @@ reloadEventHook.on(async () => { const { showAll, hideAll, saveOrUpdate } = useViewColumnsOrThrow() -const { syncLTARRefs, row } = useProvideSmartsheetRowStore( +const { state, row } = useProvideSmartsheetRowStore( meta, ref({ row: formState, @@ -124,11 +124,7 @@ async function submitForm() { if (e.errorFields.length) return } - const insertedRowData = await insertRow({ row: formState, oldRow: {}, rowMeta: { new: true } }) - - if (insertedRowData) { - await syncLTARRefs(insertedRowData) - } + await insertRow({ row: { ...formState, ...state.value }, oldRow: {}, rowMeta: { new: true } }) submitted.value = true } From 75fefa315e4958c33a1bc6acca07db6383689cdc Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Fri, 3 Nov 2023 08:21:58 +0000 Subject: [PATCH 009/167] fix: Open last opened workspace if no workspace id is present in url on first load --- packages/nc-gui/composables/useGlobal/state.ts | 1 + packages/nc-gui/composables/useGlobal/types.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/nc-gui/composables/useGlobal/state.ts b/packages/nc-gui/composables/useGlobal/state.ts index eaaf413932..92827b5fc3 100644 --- a/packages/nc-gui/composables/useGlobal/state.ts +++ b/packages/nc-gui/composables/useGlobal/state.ts @@ -67,6 +67,7 @@ export function useGlobalState(storageKey = 'nocodb-gui-v2'): State { latestRelease: null, hiddenRelease: null, isMobileMode: null, + lastUsedWorkspaceId: null, } /** saves a reactive state, any change to these values will write/delete to localStorage */ diff --git a/packages/nc-gui/composables/useGlobal/types.ts b/packages/nc-gui/composables/useGlobal/types.ts index 8549fd2e01..7cd5747345 100644 --- a/packages/nc-gui/composables/useGlobal/types.ts +++ b/packages/nc-gui/composables/useGlobal/types.ts @@ -44,6 +44,7 @@ export interface StoredState { latestRelease: string | null hiddenRelease: string | null isMobileMode: boolean | null + lastUsedWorkspaceId: string | null } export type State = ToRefs> & { From 2b5adf619baebe7f8d5f06d2be4b436615e7b285 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Fri, 3 Nov 2023 08:21:59 +0000 Subject: [PATCH 010/167] fix: Open last opened workspace if no workspace id is present in url on first load --- packages/nc-gui/composables/useGlobal/state.ts | 2 +- packages/nc-gui/composables/useGlobal/types.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nc-gui/composables/useGlobal/state.ts b/packages/nc-gui/composables/useGlobal/state.ts index 92827b5fc3..4fab0a5c72 100644 --- a/packages/nc-gui/composables/useGlobal/state.ts +++ b/packages/nc-gui/composables/useGlobal/state.ts @@ -67,7 +67,7 @@ export function useGlobalState(storageKey = 'nocodb-gui-v2'): State { latestRelease: null, hiddenRelease: null, isMobileMode: null, - lastUsedWorkspaceId: null, + lastOpenedWorkspaceId: null, } /** saves a reactive state, any change to these values will write/delete to localStorage */ diff --git a/packages/nc-gui/composables/useGlobal/types.ts b/packages/nc-gui/composables/useGlobal/types.ts index 7cd5747345..4a4f9caeb1 100644 --- a/packages/nc-gui/composables/useGlobal/types.ts +++ b/packages/nc-gui/composables/useGlobal/types.ts @@ -44,7 +44,7 @@ export interface StoredState { latestRelease: string | null hiddenRelease: string | null isMobileMode: boolean | null - lastUsedWorkspaceId: string | null + lastOpenedWorkspaceId: string | null } export type State = ToRefs> & { From 838a83b868de101266ebfdd48737482553d96518 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sun, 5 Nov 2023 00:04:42 +0530 Subject: [PATCH 011/167] fix: add max length password validation Signed-off-by: Pranav C --- packages/nocodb-sdk/src/lib/Api.ts | 24 +++++++++++++++++++++++- packages/nocodb/src/schema/swagger.json | 5 +++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/nocodb-sdk/src/lib/Api.ts b/packages/nocodb-sdk/src/lib/Api.ts index 4a080a7941..722bbb1cc5 100644 --- a/packages/nocodb-sdk/src/lib/Api.ts +++ b/packages/nocodb-sdk/src/lib/Api.ts @@ -85,6 +85,26 @@ export interface AttachmentReqType { title?: string; /** Attachment URL to be uploaded via upload-by-url */ url?: string; + /** The name of the attachment file name */ + fileName?: string; +} + +/** + * Model for File Request + */ +export interface FileReqType { + /** The mimetype of the file */ + mimetype?: string; + /** The name of the input used to upload the file */ + fieldname?: string; + /** The original name of the file */ + originalname?: string; + /** The size of the file */ + size?: number; + /** The encoding of the file */ + encoding?: string; + /** An buffer array containing the file content */ + buffer?: any; } /** @@ -10308,7 +10328,9 @@ export class Api< */ path: string; }, - data: AttachmentReqType, + data: { + files: FileReqType[]; + }, params: RequestParams = {} ) => this.request({ diff --git a/packages/nocodb/src/schema/swagger.json b/packages/nocodb/src/schema/swagger.json index 5061224070..9b4b27c2f9 100644 --- a/packages/nocodb/src/schema/swagger.json +++ b/packages/nocodb/src/schema/swagger.json @@ -20524,10 +20524,13 @@ ], "properties": { "currentPassword": { + "minLength": 8, + "maxLength": 128, "type": "string" }, "newPassword": { "minLength": 8, + "maxLength": 128, "type": "string" } }, @@ -20576,6 +20579,7 @@ "description": "New password", "example": "newpassword", "minLength": 8, + "maxLength": 128, "type": "string" } }, @@ -21596,6 +21600,7 @@ "description": "Password of the user", "example": "password123456789", "minLength": 8, + "maxLength": 128, "type": "string" }, "firstname": { From 9edc7123191432918a47f2b130d3f6fae038ddcf Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Nov 2023 07:48:03 +0000 Subject: [PATCH 012/167] refactor: error logging --- packages/nocodb/src/Noco.ts | 47 +++++++++++-------- .../global-exception.filter.ts | 6 ++- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/packages/nocodb/src/Noco.ts b/packages/nocodb/src/Noco.ts index a10c00647d..56ab6032b2 100644 --- a/packages/nocodb/src/Noco.ts +++ b/packages/nocodb/src/Noco.ts @@ -9,7 +9,7 @@ import dotenv from 'dotenv'; import { IoAdapter } from '@nestjs/platform-socket.io'; import requestIp from 'request-ip'; import cookieParser from 'cookie-parser'; -import { Logger } from '@nestjs/common'; +import type { INestApplication } from '@nestjs/common'; import type { MetaService } from '~/meta/meta.service'; import type { IEventEmitter } from '~/modules/event-emitter/event-emitter.interface'; import type { Express } from 'express'; @@ -21,17 +21,16 @@ import { isEE } from '~/utils'; dotenv.config(); export default class Noco { - private static _this: Noco; - private static ee: boolean; + protected static _this: Noco; + protected static ee: boolean; public static readonly env: string = '_noco'; - private static _httpServer: http.Server; - private static _server: Express; - private static logger = new Logger(Noco.name); + protected static _httpServer: http.Server; + protected static _server: Express; public static get dashboardUrl(): string { const siteUrl = `http://localhost:${process.env.PORT || 8080}`; - return `${siteUrl}${Noco._this?.config?.dashboardPath}`; + return `${siteUrl}${this._this?.config?.dashboardPath}`; } public static config: any; @@ -43,8 +42,8 @@ export default class Noco { public readonly metaMgrv2: any; public env: string; - private config: any; - private requestContext: any; + protected config: any; + protected requestContext: any; constructor() { process.env.PORT = process.env.PORT || '8080'; @@ -82,22 +81,26 @@ export default class Noco { } public static getConfig(): any { - return Noco.config; + return this.config; } public static isEE(): boolean { - return Noco.ee || process.env.NC_CLOUD === 'true'; + return this.ee || process.env.NC_CLOUD === 'true'; } public static async loadEEState(): Promise { try { - return (Noco.ee = isEE); + return (this.ee = isEE); } catch {} - return (Noco.ee = false); + return (this.ee = false); } static async init(param: any, httpServer: http.Server, server: Express) { - const nestApp = await NestFactory.create(AppModule); + const nestApp = await NestFactory.create(AppModule, { + bufferLogs: true, + }); + this.initCustomLogger(nestApp); + nestApp.flushLogs(); if (process.env.NC_WORKER_CONTAINER === 'true') { if (!process.env.NC_REDIS_URL) { @@ -136,23 +139,23 @@ export default class Noco { } public static get httpServer(): http.Server { - return Noco._httpServer; + return this._httpServer; } public static get server(): Express { - return Noco._server; + return this._server; } public static async initJwt(): Promise { if (this.config?.auth?.jwt) { if (!this.config.auth.jwt.secret) { let secret = ( - await Noco._ncMeta.metaGet('', '', MetaTable.STORE, { + await this._ncMeta.metaGet('', '', MetaTable.STORE, { key: 'nc_auth_jwt_secret', }) )?.value; if (!secret) { - await Noco._ncMeta.metaInsert('', '', MetaTable.STORE, { + await this._ncMeta.metaInsert('', '', MetaTable.STORE, { key: 'nc_auth_jwt_secret', value: (secret = uuidv4()), }); @@ -167,16 +170,20 @@ export default class Noco { } } let serverId = ( - await Noco._ncMeta.metaGet('', '', MetaTable.STORE, { + await this._ncMeta.metaGet('', '', MetaTable.STORE, { key: 'nc_server_id', }) )?.value; if (!serverId) { - await Noco._ncMeta.metaInsert('', '', MetaTable.STORE, { + await this._ncMeta.metaInsert('', '', MetaTable.STORE, { key: 'nc_server_id', value: (serverId = T.id), }); } process.env.NC_SERVER_UUID = serverId; } + + protected static initCustomLogger(_nestApp: INestApplication) { + // setup custom logger for nestjs if needed + } } diff --git a/packages/nocodb/src/filters/global-exception/global-exception.filter.ts b/packages/nocodb/src/filters/global-exception/global-exception.filter.ts index 9c40d418ef..1933b66e08 100644 --- a/packages/nocodb/src/filters/global-exception/global-exception.filter.ts +++ b/packages/nocodb/src/filters/global-exception/global-exception.filter.ts @@ -43,7 +43,7 @@ export class GlobalExceptionFilter implements ExceptionFilter { exception instanceof ThrottlerException ) ) - this.logger.error(exception.message, exception.stack); + this.logError(exception, request); if (exception instanceof ThrottlerException) { this.logger.log( @@ -118,4 +118,8 @@ export class GlobalExceptionFilter implements ExceptionFilter { protected captureException(exception: any, _request: any) { this.sentryClient?.instance().captureException(exception); } + + protected logError(exception: any, _request: any) { + this.logger.error(exception.message, exception.stack); + } } From b7c9553aa23acff7f0bd346cd55e7aa0dac296e2 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 6 Nov 2023 11:19:22 +0530 Subject: [PATCH 013/167] fix: use insert hook for update hook for link method --- packages/nocodb/src/db/BaseModelSqlv2.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 00e7c0ff12..79e677b876 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2524,7 +2524,7 @@ class BaseModelSqlv2 { ? JSON.parse(data[col.title]) : data[col.title]; } catch { - continue + continue; } switch (colOptions.type) { case RelationTypes.BELONGS_TO: @@ -3581,6 +3581,12 @@ class BaseModelSqlv2 { const childTn = this.getTnPath(childTable); const parentTn = this.getTnPath(parentTable); + const prevData = await this.readByPk( + rowId, + false, + {}, + { ignoreView: true, getHiddenColumn: true }, + ); switch (colOptions.type) { case RelationTypes.MANY_TO_MANY: @@ -3658,7 +3664,7 @@ class BaseModelSqlv2 { {}, { ignoreView: true, getHiddenColumn: true }, ); - await this.afterInsert(response, this.dbDriver, cookie); + await this.afterUpdate(prevData, response, this.dbDriver, cookie); await this.afterAddChild(rowId, childId, cookie); } From 84487c508769b518e57f0cafbd274d13f304dfea Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 6 Nov 2023 13:11:52 +0530 Subject: [PATCH 014/167] fix: avoid caching at the data-loader since it may leads to invalid data - in our case we are using it just for making the data processing as batch --- packages/nocodb/src/db/BaseModelSqlv2.ts | 213 +++++++++++++---------- 1 file changed, 121 insertions(+), 92 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 79e677b876..21c2604722 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -1711,28 +1711,35 @@ class BaseModelSqlv2 { (await column.getColOptions()) as LinkToAnotherRecordColumn; if (colOptions?.type === 'hm') { - const listLoader = new DataLoader(async (ids: string[]) => { - if (ids.length > 1) { - const data = await this.multipleHmList( - { - colId: column.id, - ids, - }, - (listLoader as any).args, - ); - return ids.map((id: string) => (data[id] ? data[id] : [])); - } else { - return [ - await this.hmList( + const listLoader = new DataLoader( + async (ids: string[]) => { + if (ids.length > 1) { + const data = await this.multipleHmList( { colId: column.id, - id: ids[0], + ids, }, (listLoader as any).args, - ), - ]; - } - }); + ); + return ids.map((id: string) => + data[id] ? data[id] : [], + ); + } else { + return [ + await this.hmList( + { + colId: column.id, + id: ids[0], + }, + (listLoader as any).args, + ), + ]; + } + }, + { + cache: false, + }, + ); const self: BaseModelSqlv2 = this; proto[ @@ -1746,29 +1753,34 @@ class BaseModelSqlv2 { ); }; } else if (colOptions.type === 'mm') { - const listLoader = new DataLoader(async (ids: string[]) => { - if (ids?.length > 1) { - const data = await this.multipleMmList( - { - parentIds: ids, - colId: column.id, - }, - (listLoader as any).args, - ); - - return data; - } else { - return [ - await this.mmList( + const listLoader = new DataLoader( + async (ids: string[]) => { + if (ids?.length > 1) { + const data = await this.multipleMmList( { - parentId: ids[0], + parentIds: ids, colId: column.id, }, (listLoader as any).args, - ), - ]; - } - }); + ); + + return data; + } else { + return [ + await this.mmList( + { + parentId: ids[0], + colId: column.id, + }, + (listLoader as any).args, + ), + ]; + } + }, + { + cache: false, + }, + ); const self: BaseModelSqlv2 = this; proto[ @@ -1796,55 +1808,62 @@ class BaseModelSqlv2 { // it takes individual keys and callback is invoked with an array of values and we can get the // result for all those together and return the value in the same order as in the array // this way all parents data extracted together - const readLoader = new DataLoader(async (_ids: string[]) => { - // handle binary(16) foreign keys - const ids = _ids.map((id) => { - if (pCol.ct !== 'binary(16)') return id; - - // Cast the id to string. - const idAsString = id + ''; - // Check if the id is a UUID and the column is binary(16) - const isUUIDBinary16 = - idAsString.length === 36 || idAsString.length === 32; - // If the id is a UUID and the column is binary(16), convert the id to a Buffer. Otherwise, return null to indicate that the id is not a UUID. - const idAsUUID = isUUIDBinary16 - ? idAsString.length === 32 - ? idAsString.replace( - /(.{8})(.{4})(.{4})(.{4})(.{12})/, - '$1-$2-$3-$4-$5', - ) - : idAsString - : null; - - return idAsUUID - ? Buffer.from(idAsUUID.replace(/-/g, ''), 'hex') - : id; - }); - - const data = await ( - await Model.getBaseModelSQL({ - id: pCol.fk_model_id, - dbDriver: this.dbDriver, - }) - ).list( - { - fieldsSet: (readLoader as any).args?.fieldsSet, - filterArr: [ - new Filter({ - id: null, - fk_column_id: pCol.id, - fk_model_id: pCol.fk_model_id, - value: ids as any[], - comparison_op: 'in', - }), - ], - }, - true, - ); + const readLoader = new DataLoader( + async (_ids: string[]) => { + // handle binary(16) foreign keys + const ids = _ids.map((id) => { + if (pCol.ct !== 'binary(16)') return id; + + // Cast the id to string. + const idAsString = id + ''; + // Check if the id is a UUID and the column is binary(16) + const isUUIDBinary16 = + idAsString.length === 36 || idAsString.length === 32; + // If the id is a UUID and the column is binary(16), convert the id to a Buffer. Otherwise, return null to indicate that the id is not a UUID. + const idAsUUID = isUUIDBinary16 + ? idAsString.length === 32 + ? idAsString.replace( + /(.{8})(.{4})(.{4})(.{4})(.{12})/, + '$1-$2-$3-$4-$5', + ) + : idAsString + : null; + + return idAsUUID + ? Buffer.from(idAsUUID.replace(/-/g, ''), 'hex') + : id; + }); + + const data = await ( + await Model.getBaseModelSQL({ + id: pCol.fk_model_id, + dbDriver: this.dbDriver, + }) + ).list( + { + fieldsSet: (readLoader as any).args?.fieldsSet, + filterArr: [ + new Filter({ + id: null, + fk_column_id: pCol.id, + fk_model_id: pCol.fk_model_id, + value: ids as any[], + comparison_op: 'in', + }), + ], + }, + true, + ); - const groupedList = groupBy(data, pCol.title); - return _ids.map(async (id: string) => groupedList?.[id]?.[0]); - }); + const groupedList = groupBy(data, pCol.title); + return _ids.map( + async (id: string) => groupedList?.[id]?.[0], + ); + }, + { + cache: false, + }, + ); // defining BelongsTo read resolver method proto[column.title] = async function (args?: any) { @@ -4332,7 +4351,7 @@ class BaseModelSqlv2 { } async addLinks({ - cookie: _cookie, + cookie, childIds, colId, rowId, @@ -4348,9 +4367,12 @@ class BaseModelSqlv2 { if (!column || !isLinksOrLTAR(column)) NcError.notFound(`Link column ${colId} not found`); - const row = await this.dbDriver(this.tnPath) - .where(await this._wherePk(rowId)) - .first(); + const row = await this.readByPk( + rowId, + false, + {}, + { ignoreView: true, getHiddenColumn: true }, + ); // validate rowId if (!row) { @@ -4554,9 +4576,16 @@ class BaseModelSqlv2 { break; } - // const response = await this.readByPk(rowId); - // await this.afterInsert(response, this.dbDriver, cookie); - // await this.afterAddChild(rowId, childId, cookie); + const response = await this.readByPk( + rowId, + false, + {}, + { ignoreView: true, getHiddenColumn: true }, + ); + await this.afterUpdate(row, response, this.dbDriver, cookie); + for (const childId of childIds) { + await this.afterAddChild(rowId, childId, cookie); + } } async removeLinks({ From 210aedbadff93abc43d5860689cdecec6acf025d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 6 Nov 2023 13:14:56 +0530 Subject: [PATCH 015/167] fix: new data api links correction --- packages/nocodb/src/db/BaseModelSqlv2.ts | 28 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 21c2604722..f0fd030ef0 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -4576,20 +4576,20 @@ class BaseModelSqlv2 { break; } - const response = await this.readByPk( + const updatedRow = await this.readByPk( rowId, false, {}, { ignoreView: true, getHiddenColumn: true }, ); - await this.afterUpdate(row, response, this.dbDriver, cookie); + await this.afterUpdate(row, updatedRow, this.dbDriver, cookie); for (const childId of childIds) { await this.afterAddChild(rowId, childId, cookie); } } async removeLinks({ - cookie: _cookie, + cookie, childIds, colId, rowId, @@ -4605,9 +4605,12 @@ class BaseModelSqlv2 { if (!column || !isLinksOrLTAR(column)) NcError.notFound(`Link column ${colId} not found`); - const row = await this.dbDriver(this.tnPath) - .where(await this._wherePk(rowId)) - .first(); + const row = await this.readByPk( + rowId, + false, + {}, + { ignoreView: true, getHiddenColumn: true }, + ); // validate rowId if (!row) { @@ -4786,9 +4789,16 @@ class BaseModelSqlv2 { break; } - // const newData = await this.readByPk(rowId); - // await this.afterUpdate(prevData, newData, this.dbDriver, cookie); - // await this.afterRemoveChild(rowId, childIds, cookie); + const updatedRow = await this.readByPk( + rowId, + false, + {}, + { ignoreView: true, getHiddenColumn: true }, + ); + await this.afterUpdate(row, updatedRow, this.dbDriver, cookie); + for (const childId of childIds) { + await this.afterRemoveChild(rowId, childId, cookie); + } } async btRead( From 265839a9b7aae9799e6d720fbf90548fbe267497 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Mon, 6 Nov 2023 09:34:27 +0000 Subject: [PATCH 016/167] fix: Ignore system column created with table create camelisation, when used on data source with no inflection --- packages/nc-gui/composables/useTableNew.ts | 30 +++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/nc-gui/composables/useTableNew.ts b/packages/nc-gui/composables/useTableNew.ts index 4f8865c9fe..d514116769 100644 --- a/packages/nc-gui/composables/useTableNew.ts +++ b/packages/nc-gui/composables/useTableNew.ts @@ -113,17 +113,29 @@ export function useTableNew(param: { onTableCreate?: (tableMeta: TableType) => v } const sqlUi = await basesStore.getSqlUi(baseId, sourceId) + const source = bases.value.get(baseId)?.sources?.find((s) => s.id === sourceId) if (!sqlUi) return - const columns = sqlUi?.getNewTableColumns().filter((col: ColumnType) => { - if (col.column_name === 'id' && table.columns.includes('id_ag')) { - Object.assign(col, sqlUi?.getDataTypeForUiType({ uidt: UITypes.ID }, 'AG')) - col.dtxp = sqlUi?.getDefaultLengthForDatatype(col.dt) - col.dtxs = sqlUi?.getDefaultScaleForDatatype(col.dt) - return true - } - return table.columns.includes(col.column_name!) - }) + const columns = sqlUi + ?.getNewTableColumns() + .filter((col: ColumnType) => { + if (col.column_name === 'id' && table.columns.includes('id_ag')) { + Object.assign(col, sqlUi?.getDataTypeForUiType({ uidt: UITypes.ID }, 'AG')) + col.dtxp = sqlUi?.getDefaultLengthForDatatype(col.dt) + col.dtxs = sqlUi?.getDefaultScaleForDatatype(col.dt) + return true + } + return table.columns.includes(col.column_name!) + }) + .map((column) => { + if (!source) return column + + if (source.inflection_column !== 'camelize') { + column.title = column.column_name + } + + return column + }) try { const tableMeta = await $api.source.tableCreate(baseId, sourceId!, { From 123e54b485e2dc705b820ec40640e00b4c38464e Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Mon, 6 Nov 2023 09:34:27 +0000 Subject: [PATCH 017/167] fix: Made default inflection as none for external data source --- .../dashboard/settings/data-sources/CreateBase.vue | 9 ++++----- .../dashboard/settings/data-sources/EditBase.vue | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/nc-gui/components/dashboard/settings/data-sources/CreateBase.vue b/packages/nc-gui/components/dashboard/settings/data-sources/CreateBase.vue index 371ca85ed2..a62f6b0940 100644 --- a/packages/nc-gui/components/dashboard/settings/data-sources/CreateBase.vue +++ b/packages/nc-gui/components/dashboard/settings/data-sources/CreateBase.vue @@ -66,8 +66,8 @@ const formState = ref({ title: '', dataSource: { ...getDefaultConnectionConfig(ClientType.MYSQL) }, inflection: { - inflectionColumn: 'camelize', - inflectionTable: 'camelize', + inflectionColumn: 'none', + inflectionTable: 'none', }, sslUse: SSLUsage.No, extraParameters: [], @@ -77,8 +77,8 @@ const customFormState = ref({ title: '', dataSource: { ...getDefaultConnectionConfig(ClientType.MYSQL) }, inflection: { - inflectionColumn: 'camelize', - inflectionTable: 'camelize', + inflectionColumn: 'none', + inflectionTable: 'none', }, sslUse: SSLUsage.No, extraParameters: [], @@ -614,7 +614,6 @@ const toggleModal = (val: boolean) => { - ({ title: '', dataSource: { ...getDefaultConnectionConfig(ClientType.MYSQL) }, inflection: { - inflectionColumn: 'camelize', - inflectionTable: 'camelize', + inflectionColumn: 'none', + inflectionTable: 'none', }, sslUse: SSLUsage.No, extraParameters: [], @@ -72,8 +72,8 @@ const customFormState = ref({ title: '', dataSource: { ...getDefaultConnectionConfig(ClientType.MYSQL) }, inflection: { - inflectionColumn: 'camelize', - inflectionTable: 'camelize', + inflectionColumn: 'none', + inflectionTable: 'none', }, sslUse: SSLUsage.No, extraParameters: [], From 82c00676f5aa7d0e83ef36613f2e9df0481aed61 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Mon, 6 Nov 2023 11:31:37 +0000 Subject: [PATCH 018/167] fix: Hide created column for all other view except for the view it created --- .../nc-gui/components/smartsheet/header/Menu.vue | 1 + .../nc-gui/composables/useColumnCreateStore.ts | 8 +++++++- packages/nocodb/src/models/Column.ts | 14 +++++++++++++- packages/nocodb/src/schema/swagger.json | 3 +++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/header/Menu.vue b/packages/nc-gui/components/smartsheet/header/Menu.vue index d364d863c9..af980dcd59 100644 --- a/packages/nc-gui/components/smartsheet/header/Menu.vue +++ b/packages/nc-gui/components/smartsheet/header/Menu.vue @@ -170,6 +170,7 @@ const duplicateColumn = async () => { await $api.dbTableColumn.create(meta!.value!.id!, { ...columnCreatePayload, pv: false, + view_id: view.value!.id as string, column_order: { order: newColumnOrder, view_id: view.value?.id as string, diff --git a/packages/nc-gui/composables/useColumnCreateStore.ts b/packages/nc-gui/composables/useColumnCreateStore.ts index cb7b634259..f01bde2160 100644 --- a/packages/nc-gui/composables/useColumnCreateStore.ts +++ b/packages/nc-gui/composables/useColumnCreateStore.ts @@ -50,6 +50,8 @@ const [useProvideColumnCreateStore, useColumnCreateStore] = createInjectionState const sqlUi = ref(meta.value?.source_id ? sqlUis.value[meta.value?.source_id] : Object.values(sqlUis.value)[0]) + const { activeView } = storeToRefs(useViewsStore()) + const isEdit = computed(() => !!column?.value?.id) const isMysql = computed(() => isMysqlFunc(meta.value?.source_id ? meta.value?.source_id : Object.keys(sqlUis.value)[0])) @@ -275,7 +277,11 @@ const [useProvideColumnCreateStore, useColumnCreateStore] = createInjectionState // }; // } } - await $api.dbTableColumn.create(meta.value?.id as string, { ...formState.value, ...columnPosition }) + await $api.dbTableColumn.create(meta.value?.id as string, { + ...formState.value, + ...columnPosition, + view_id: activeView.value!.id as string, + }) /** if LTAR column then force reload related table meta */ if (isLinksOrLTAR(formState.value) && meta.value?.id !== formState.value.childId) { diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts index 7fcabb551d..d6d17e9454 100644 --- a/packages/nocodb/src/models/Column.ts +++ b/packages/nocodb/src/models/Column.ts @@ -99,6 +99,7 @@ export default class Column implements ColumnType { [key: string]: any; fk_model_id: string; uidt: UITypes | string; + view_id?: string; } & Pick, ncMeta = Noco.ncMeta, ) { @@ -185,7 +186,7 @@ export default class Column implements ColumnType { { fk_column_id: row.id, fk_model_id: column.fk_model_id, - show: true, + show: false, column_order: column.column_order, }, ncMeta, @@ -196,6 +197,17 @@ export default class Column implements ColumnType { `${column.fk_model_id}:default:*`, ); + if (column.view_id) { + await View.insertOrUpdateColumn( + column.view_id, + row.id, + { + show: true, + }, + ncMeta, + ); + } + return col; } diff --git a/packages/nocodb/src/schema/swagger.json b/packages/nocodb/src/schema/swagger.json index 88ea466c72..4eabecb8a4 100644 --- a/packages/nocodb/src/schema/swagger.json +++ b/packages/nocodb/src/schema/swagger.json @@ -17427,6 +17427,9 @@ "maxLength": 255, "minLength": 1, "type": "string" + }, + "view_id": { + "type": "string" } }, "required": [ From fb9642b0bc8667ea3acb89f50ebe819f78c88082 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Mon, 6 Nov 2023 11:31:37 +0000 Subject: [PATCH 019/167] fix: Added getViewColumnId and fixed issue with view column creation --- packages/nocodb/src/models/Column.ts | 18 +++++++-- packages/nocodb/src/models/View.ts | 58 ++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts index d6d17e9454..93664591cf 100644 --- a/packages/nocodb/src/models/Column.ts +++ b/packages/nocodb/src/models/Column.ts @@ -198,14 +198,24 @@ export default class Column implements ColumnType { ); if (column.view_id) { - await View.insertOrUpdateColumn( - column.view_id, - row.id, + const viewColId = await View.getViewColumnId( { - show: true, + viewId: column.view_id, + colId: row.id, }, ncMeta, ); + + if (viewColId) { + await View.updateColumn( + column.view_id, + viewColId, + { + show: true, + }, + ncMeta, + ); + } } return col; diff --git a/packages/nocodb/src/models/View.ts b/packages/nocodb/src/models/View.ts index 521968b815..0ac2cf30b2 100644 --- a/packages/nocodb/src/models/View.ts +++ b/packages/nocodb/src/models/View.ts @@ -680,6 +680,64 @@ export default class View implements ViewType { return (this.columns = await View.getColumns(this.id, ncMeta)); } + static async getViewColumnId( + { + viewId, + colId, + }: { + viewId: string; + colId: string; + }, + ncMeta = Noco.ncMeta, + ) { + const view = await this.get(viewId); + if (!view) return undefined; + + let tableName; + let cacheScope; + switch (view.type) { + case ViewTypes.GRID: + tableName = MetaTable.GRID_VIEW_COLUMNS; + cacheScope = CacheScope.GRID_VIEW_COLUMN; + + break; + case ViewTypes.GALLERY: + tableName = MetaTable.GALLERY_VIEW_COLUMNS; + cacheScope = CacheScope.GALLERY_VIEW_COLUMN; + + break; + case ViewTypes.MAP: + tableName = MetaTable.MAP_VIEW_COLUMNS; + cacheScope = CacheScope.MAP_VIEW_COLUMN; + + break; + case ViewTypes.FORM: + tableName = MetaTable.FORM_VIEW_COLUMNS; + cacheScope = CacheScope.FORM_VIEW_COLUMN; + + break; + case ViewTypes.KANBAN: + tableName = MetaTable.KANBAN_VIEW_COLUMNS; + cacheScope = CacheScope.KANBAN_VIEW_COLUMN; + + break; + } + + const key = `${cacheScope}:viewColumnId:${colId}`; + const o = await NocoCache.get(key, CacheGetType.TYPE_STRING); + if (o) return o; + + const viewColumn = await ncMeta.metaGet2(null, null, tableName, { + fk_view_id: viewId, + fk_column_id: colId, + }); + if (!viewColumn) return undefined; + + await NocoCache.set(key, viewColumn.id); + + return viewColumn?.id; + } + static async updateColumn( viewId: string, colId: string, From adbfaeff02d0d9813c92fb1079ff02c52d676ba3 Mon Sep 17 00:00:00 2001 From: Muhammed Mustafa Date: Mon, 6 Nov 2023 11:31:37 +0000 Subject: [PATCH 020/167] fix: Made column hide by default only when view id is explicitly given to the create column model --- packages/nocodb/src/models/Column.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts index 93664591cf..6977e28940 100644 --- a/packages/nocodb/src/models/Column.ts +++ b/packages/nocodb/src/models/Column.ts @@ -186,7 +186,7 @@ export default class Column implements ColumnType { { fk_column_id: row.id, fk_model_id: column.fk_model_id, - show: false, + show: column.view_id ? false : true, column_order: column.column_order, }, ncMeta, From 683cb68f426ecfada5eef34edc4a9fa4afbd60d4 Mon Sep 17 00:00:00 2001 From: gitstart Date: Mon, 6 Nov 2023 11:43:24 +0000 Subject: [PATCH 021/167] feat: update datetimepicker start day Co-authored-by: gitstart Co-authored-by: frankmagoba Co-authored-by: gitstart_bot --- packages/nc-gui/components/cell/DateTimePicker.vue | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/nc-gui/components/cell/DateTimePicker.vue b/packages/nc-gui/components/cell/DateTimePicker.vue index 984806908e..3181047504 100644 --- a/packages/nc-gui/components/cell/DateTimePicker.vue +++ b/packages/nc-gui/components/cell/DateTimePicker.vue @@ -1,6 +1,7 @@