From 0417aa8f16e62a0a2cce1dedf67f5a90e71fc4c4 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 19 Feb 2024 05:12:40 +0000 Subject: [PATCH 1/9] fix: support composite key in v2 update --- packages/nocodb/src/db/BaseModelSqlv2.ts | 38 ++++++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index b6a7c16034..c8704e0244 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2870,6 +2870,22 @@ class BaseModelSqlv2 { : response?.[ai.id]; } + // handle if composite primary key is used along with ai or ag + if ((ai || ag) && this.model.primaryKeys?.length > 1) { + // generate object with ai column and rest of the primary keys + const pkObj = {}; + for (const pk of this.model.primaryKeys) { + if (ai && pk.id === ai.id) { + pkObj[pk.id] = rowId; + } else if (ag && pk.id === ag.id) { + pkObj[pk.id] = rowId; + } else { + pkObj[pk.id] = insertObj[pk.column_name] ?? null; + } + } + rowId = pkObj; + } + await Promise.all(postInsertOps.map((f) => f(rowId))); if (rowId !== null && rowId !== undefined) { @@ -3892,12 +3908,22 @@ class BaseModelSqlv2 { // todo: handle composite primary key protected _extractPksValues(data: any) { // data can be still inserted without PK - // TODO: return a meaningful value - if (!this.model.primaryKey) return 'N/A'; - return ( - data[this.model.primaryKey.title] || - data[this.model.primaryKey.column_name] - ); + + // if composite primary key return an object with all the primary keys + if (this.model.primaryKeys.length > 1) { + const pkValues = {}; + for (const pk of this.model.primaryKeys) { + pkValues[pk.title] = data[pk.title] || data[pk.column_name]; + } + return pkValues; + } else if (this.model.primaryKey) { + return ( + data[this.model.primaryKey.title] || + data[this.model.primaryKey.column_name] + ); + } else { + return 'N/A'; + } } protected async errorDelete(_e, _id, _trx, _cookie) {} From 514ca8835f58757fa986150519c9a20fced97bdd Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 19 Feb 2024 05:12:41 +0000 Subject: [PATCH 2/9] refactor: give auto generated key priority among primary keys --- packages/nocodb/src/models/Model.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/models/Model.ts b/packages/nocodb/src/models/Model.ts index af9551819d..3a0ab05696 100644 --- a/packages/nocodb/src/models/Model.ts +++ b/packages/nocodb/src/models/Model.ts @@ -80,7 +80,12 @@ export default class Model implements TableType { public get primaryKey(): Column { if (!this.columns) return null; - return this.columns?.find((c) => c.pk); + // return first auto increment or augto generated column + // if not found return first pk column + return ( + this.columns.find((c) => c.pk && (c.ai || c.meta?.ag)) || + this.columns?.find((c) => c.pk) + ); } public get primaryKeys(): Column[] { From 0ef551b853b9d8a7402772ddeda39e1c0aee1598 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 19 Feb 2024 05:12:41 +0000 Subject: [PATCH 3/9] refactor: use composite key extraction in bulk api implementations --- packages/nocodb/src/db/BaseModelSqlv2.ts | 100 ++++++++++++++++------- 1 file changed, 72 insertions(+), 28 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index c8704e0244..66c0b2d5b8 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2449,7 +2449,11 @@ class BaseModelSqlv2 { if (ag) { if (!response) await this.execAndParse(query); response = await this.readByPk( - insertObj[ag.column_name], + this.extractCompositePK({ + rowId: insertObj[ag.column_name], + insertObj, + ag, + }), false, {}, { ignoreView: true, getHiddenColumn: true }, @@ -2492,7 +2496,7 @@ class BaseModelSqlv2 { ).id; } response = await this.readByPk( - id, + this.extractCompositePK({ rowId: id, insertObj, ag }), false, {}, { ignoreView: true, getHiddenColumn: true }, @@ -2505,7 +2509,7 @@ class BaseModelSqlv2 { ? response?.[0]?.[ai.id] : response?.[ai.id]; response = await this.readByPk( - id, + this.extractCompositePK({ rowId: id, insertObj, ag }), false, {}, { ignoreView: true, getHiddenColumn: true }, @@ -2869,22 +2873,7 @@ class BaseModelSqlv2 { ? response?.[0]?.[ai.id] : response?.[ai.id]; } - - // handle if composite primary key is used along with ai or ag - if ((ai || ag) && this.model.primaryKeys?.length > 1) { - // generate object with ai column and rest of the primary keys - const pkObj = {}; - for (const pk of this.model.primaryKeys) { - if (ai && pk.id === ai.id) { - pkObj[pk.id] = rowId; - } else if (ag && pk.id === ag.id) { - pkObj[pk.id] = rowId; - } else { - pkObj[pk.id] = insertObj[pk.column_name] ?? null; - } - } - rowId = pkObj; - } + rowId = this.extractCompositePK({ ai, ag, rowId, insertObj }); await Promise.all(postInsertOps.map((f) => f(rowId))); @@ -2905,6 +2894,35 @@ class BaseModelSqlv2 { } } + private extractCompositePK({ + ai, + ag, + rowId, + insertObj, + }: { + ai?: Column; + ag?: Column; + rowId; + insertObj: Record; + }) { + // handle if composite primary key is used along with ai or ag + if ((ai || ag) && this.model.primaryKeys?.length > 1) { + // generate object with ai column and rest of the primary keys + const pkObj = {}; + for (const pk of this.model.primaryKeys) { + if (ai && pk.id === ai.id) { + pkObj[pk.id] = rowId; + } else if (ag && pk.id === ag.id) { + pkObj[pk.id] = rowId; + } else { + pkObj[pk.id] = insertObj[pk.column_name] ?? null; + } + } + rowId = pkObj; + } + return rowId; + } + private async prepareNestedLinkQb({ nestedCols, data, @@ -3198,27 +3216,43 @@ class BaseModelSqlv2 { let responses; + const aiPkCol = this.model.primaryKeys.find((pk) => pk.ai); + const agPkCol = this.model.primaryKeys.find((pk) => pk.meta?.ag); + // insert one by one as fallback to get ids for sqlite and mysql if (insertOneByOneAsFallback && (this.isSqlite || this.isMySQL)) { // sqlite and mysql doesn't support returning, so insert one by one and return ids responses = []; - const aiPkCol = this.model.primaryKeys.find((pk) => pk.ai); - for (const insertData of insertDatas) { const query = trx(this.tnPath).insert(insertData); - const id = (await query)[0]; - responses.push(aiPkCol ? { [aiPkCol.title]: id } : id); + let id = (await query)[0]; + + if (agPkCol) { + id = insertData[agPkCol.column_name]; + } + + responses.push( + this.extractCompositePK({ + rowId: id, + ai: aiPkCol, + ag: agPkCol, + insertObj: insertData, + }) || insertData, + ); } } else { + const returningObj: Record = {}; + + for (const col of this.model.primaryKeys) { + returningObj[col.title] = col.column_name; + } + responses = !raw && (this.isPg || this.isMssql) ? await trx .batchInsert(this.tnPath, insertDatas, chunkSize) - .returning({ - [this.model.primaryKey?.title]: - this.model.primaryKey?.column_name, - }) + .returning(returningObj) : await trx.batchInsert(this.tnPath, insertDatas, chunkSize); } @@ -3232,7 +3266,17 @@ class BaseModelSqlv2 { // insert nested link data for single record insertion if (isSingleRecordInsertion) { - const rowId = responses[0][this.model.primaryKey?.title]; + let rowId = responses[0][this.model.primaryKey?.title]; + + if (aiPkCol || agPkCol) { + rowId = this.extractCompositePK({ + rowId, + ai: aiPkCol, + ag: agPkCol, + insertObj: insertDatas[0], + }); + } + await Promise.all(postInsertOps.map((f) => f(rowId, trx))); } From a8df9d25ca9bd8ba1ce4ea24d6d178e8ef90b9e8 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 19 Feb 2024 05:12:41 +0000 Subject: [PATCH 4/9] feat: add composite support in baseModel ee version --- packages/nocodb/src/db/BaseModelSqlv2.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 66c0b2d5b8..650f2984ef 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2450,7 +2450,8 @@ class BaseModelSqlv2 { if (!response) await this.execAndParse(query); response = await this.readByPk( this.extractCompositePK({ - rowId: insertObj[ag.column_name], + rowId: + this.extractCompositePK({ rowId: insertObj[ag.column_name], insertObj, ag }), insertObj, ag, }), @@ -2894,7 +2895,7 @@ class BaseModelSqlv2 { } } - private extractCompositePK({ + protected extractCompositePK({ ai, ag, rowId, From 7898d687907f6577deb3aa3a12af32d2198ddd0b Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 19 Feb 2024 05:12:41 +0000 Subject: [PATCH 5/9] feat: add composite support in baseModel ee version - bulkinsert with nested data --- packages/nocodb/src/db/BaseModelSqlv2.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 650f2984ef..419c741de0 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2924,7 +2924,7 @@ class BaseModelSqlv2 { return rowId; } - private async prepareNestedLinkQb({ + protected async prepareNestedLinkQb({ nestedCols, data, insertObj, @@ -3243,17 +3243,17 @@ class BaseModelSqlv2 { ); } } else { - const returningObj: Record = {}; + const returningArr: string[] = []; for (const col of this.model.primaryKeys) { - returningObj[col.title] = col.column_name; + returningArr.push(col.column_name); } responses = !raw && (this.isPg || this.isMssql) ? await trx .batchInsert(this.tnPath, insertDatas, chunkSize) - .returning(returningObj) + .returning(this.model.primaryKeys?.length ? returningArr : '*') : await trx.batchInsert(this.tnPath, insertDatas, chunkSize); } @@ -4025,7 +4025,7 @@ class BaseModelSqlv2 { } // method for validating otpions if column is single/multi select - private async validateOptions( + protected async validateOptions( column: Column, insertOrUpdateObject: Record, ) { From 281a67990675bf1767ca9b858a029d7b4c7609b5 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 19 Feb 2024 16:29:10 +0530 Subject: [PATCH 6/9] fix: extract as object Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 419c741de0..07fc9237a5 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2450,8 +2450,11 @@ class BaseModelSqlv2 { if (!response) await this.execAndParse(query); response = await this.readByPk( this.extractCompositePK({ - rowId: - this.extractCompositePK({ rowId: insertObj[ag.column_name], insertObj, ag }), + rowId: this.extractCompositePK({ + rowId: insertObj[ag.column_name], + insertObj, + ag, + }), insertObj, ag, }), @@ -2900,14 +2903,16 @@ class BaseModelSqlv2 { ag, rowId, insertObj, + force = false, }: { ai?: Column; ag?: Column; rowId; insertObj: Record; + force?: boolean; }) { // handle if composite primary key is used along with ai or ag - if ((ai || ag) && this.model.primaryKeys?.length > 1) { + if ((ai || ag) && (force || this.model.primaryKeys?.length > 1)) { // generate object with ai column and rest of the primary keys const pkObj = {}; for (const pk of this.model.primaryKeys) { @@ -3239,6 +3244,7 @@ class BaseModelSqlv2 { ai: aiPkCol, ag: agPkCol, insertObj: insertData, + force: true, }) || insertData, ); } From 6e522b0a047353009f00ea3936782e7ca7f78cc0 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 19 Feb 2024 16:30:41 +0530 Subject: [PATCH 7/9] fix: remove extra extractCompositePK call Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 07fc9237a5..2a6adeb4e7 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2450,11 +2450,7 @@ class BaseModelSqlv2 { if (!response) await this.execAndParse(query); response = await this.readByPk( this.extractCompositePK({ - rowId: this.extractCompositePK({ - rowId: insertObj[ag.column_name], - insertObj, - ag, - }), + rowId: insertObj[ag.column_name], insertObj, ag, }), From fb722a50377b6a58f1d90213049e98af55b8636f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 19 Feb 2024 16:46:00 +0530 Subject: [PATCH 8/9] fix: extract as title Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 2a6adeb4e7..08823aa79e 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2912,12 +2912,13 @@ class BaseModelSqlv2 { // generate object with ai column and rest of the primary keys const pkObj = {}; for (const pk of this.model.primaryKeys) { + const key = pk.id; if (ai && pk.id === ai.id) { - pkObj[pk.id] = rowId; + pkObj[key] = rowId; } else if (ag && pk.id === ag.id) { - pkObj[pk.id] = rowId; + pkObj[key] = rowId; } else { - pkObj[pk.id] = insertObj[pk.column_name] ?? null; + pkObj[key] = insertObj[pk.column_name] ?? null; } } rowId = pkObj; From 62ef20a6ee1b9ab41afb0124f46c73bba420dd16 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 19 Feb 2024 17:04:08 +0530 Subject: [PATCH 9/9] fix: extract as title Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 08823aa79e..6bac3a9673 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2912,7 +2912,7 @@ class BaseModelSqlv2 { // generate object with ai column and rest of the primary keys const pkObj = {}; for (const pk of this.model.primaryKeys) { - const key = pk.id; + const key = pk.title; if (ai && pk.id === ai.id) { pkObj[key] = rowId; } else if (ag && pk.id === ag.id) {