From d9b03d3526c1ed617a56874ec74d13d4c2b41629 Mon Sep 17 00:00:00 2001 From: mertmit Date: Wed, 22 Nov 2023 11:40:44 +0000 Subject: [PATCH] feat: use id instead of alias on queries --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 2 +- packages/nocodb/src/db/BaseModelSqlv2.ts | 174 +++++++++++------- 2 files changed, 106 insertions(+), 70 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 5a28417d84..a5a15ed532 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -93,7 +93,7 @@ export function substituteColumnIdWithAliasInFormula( c.column_name === colNameOrId || c.title === colNameOrId ); - pt.name = column?.title || ptRaw?.name || pt?.name; + pt.name = column?.id || ptRaw?.name || pt?.name; } else if (pt.type === 'BinaryExpression') { substituteId(pt.left, ptRaw?.left); substituteId(pt.right, ptRaw?.right); diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index a60d09856d..089d5fc586 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -167,7 +167,9 @@ class BaseModelSqlv2 { let data; try { - data = await this.execAndParse(qb, null, { first: true }); + data = await this.execAndParse(qb, null, { + first: true, + }); } catch (e) { if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) throw e; @@ -587,9 +589,9 @@ class BaseModelSqlv2 { knex: this.dbDriver, columnOptions: (await column.getColOptions()) as RollupColumn, }) - ).builder.as(sanitize(column.title)), + ).builder.as(sanitize(column.id)), ); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); break; case UITypes.Formula: { @@ -601,18 +603,18 @@ class BaseModelSqlv2 { selectQb = this.dbDriver.raw(`?? as ??`, [ _selectQb.builder, - sanitize(column.title), + sanitize(column.id), ]); } catch (e) { console.log(e); // return dummy select selectQb = this.dbDriver.raw(`'ERR' as ??`, [ - sanitize(column.title), + sanitize(column.id), ]); } selectors.push(selectQb); - groupBySelectors.push(column.title); + groupBySelectors.push(column.id); } break; case UITypes.Lookup: @@ -628,18 +630,18 @@ class BaseModelSqlv2 { const selectQb = this.dbDriver.raw(`?? as ??`, [ this.dbDriver.raw(_selectQb.builder).wrap('(', ')'), - sanitize(column.title), + sanitize(column.id), ]); selectors.push(selectQb); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); } break; default: selectors.push( - this.dbDriver.raw('?? as ??', [column.column_name, column.title]), + this.dbDriver.raw('?? as ??', [column.column_name, column.id]), ); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); break; } }), @@ -704,7 +706,7 @@ class BaseModelSqlv2 { } qb.orderBy( - groupByColumns[sort.fk_column_id].title, + groupByColumns[sort.fk_column_id].id, sort.direction, sort.direction === 'desc' ? 'LAST' : 'FIRST', ); @@ -772,9 +774,9 @@ class BaseModelSqlv2 { columnOptions: (await column.getColOptions()) as RollupColumn, }) - ).builder.as(sanitize(column.title)), + ).builder.as(sanitize(column.id)), ); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); break; case UITypes.Formula: { let selectQb; @@ -785,18 +787,18 @@ class BaseModelSqlv2 { selectQb = this.dbDriver.raw(`?? as ??`, [ _selectQb.builder, - sanitize(column.title), + sanitize(column.id), ]); } catch (e) { console.log(e); // return dummy select selectQb = this.dbDriver.raw(`'ERR' as ??`, [ - sanitize(column.title), + sanitize(column.id), ]); } selectors.push(selectQb); - groupBySelectors.push(column.title); + groupBySelectors.push(column.id); break; } case UITypes.Lookup: @@ -812,21 +814,18 @@ class BaseModelSqlv2 { const selectQb = this.dbDriver.raw(`?? as ??`, [ this.dbDriver.raw(_selectQb.builder).wrap('(', ')'), - sanitize(column.title), + sanitize(column.id), ]); selectors.push(selectQb); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); } break; default: selectors.push( - this.dbDriver.raw('?? as ??', [ - column.column_name, - column.title, - ]), + this.dbDriver.raw('?? as ??', [column.column_name, column.id]), ); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); break; } }), @@ -2099,11 +2098,10 @@ class BaseModelSqlv2 { // the value 2023-01-01 10:00:00 (UTC) would display as 2023-01-01 18:00:00 (UTC+8) // our existing logic is based on UTC, during the query, we need to take the UTC value // hence, we use CONVERT_TZ to convert back to UTC value - res[sanitize(column.title || column.column_name)] = - this.dbDriver.raw( - `CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00')`, - [`${sanitize(alias || this.tnPath)}.${column.column_name}`], - ); + res[sanitize(column.id || column.column_name)] = this.dbDriver.raw( + `CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00')`, + [`${sanitize(alias || this.tnPath)}.${column.column_name}`], + ); break; } else if (this.isPg) { // if there is no timezone info, @@ -2113,7 +2111,7 @@ class BaseModelSqlv2 { column.dt !== 'timestamp with time zone' && column.dt !== 'timestamptz' ) { - res[sanitize(column.title || column.column_name)] = this.dbDriver + res[sanitize(column.id || column.column_name)] = this.dbDriver .raw( `?? AT TIME ZONE CURRENT_SETTING('timezone') AT TIME ZONE 'UTC'`, [`${sanitize(alias || this.tnPath)}.${column.column_name}`], @@ -2126,7 +2124,7 @@ class BaseModelSqlv2 { // convert to database timezone, // then convert to UTC if (column.dt !== 'datetimeoffset') { - res[sanitize(column.title || column.column_name)] = + res[sanitize(column.id || column.column_name)] = this.dbDriver.raw( `CONVERT(DATETIMEOFFSET, ?? AT TIME ZONE 'UTC')`, [`${sanitize(alias || this.tnPath)}.${column.column_name}`], @@ -2134,7 +2132,7 @@ class BaseModelSqlv2 { break; } } - res[sanitize(column.title || column.column_name)] = sanitize( + res[sanitize(column.id || column.column_name)] = sanitize( `${alias || this.tnPath}.${column.column_name}`, ); break; @@ -2197,7 +2195,7 @@ class BaseModelSqlv2 { aliasToColumnBuilder, ); qb.select({ - [column.title]: selectQb.builder, + [column.id]: selectQb.builder, }); } catch { continue; @@ -2205,7 +2203,7 @@ class BaseModelSqlv2 { break; default: { qb.select({ - [column.title]: barcodeValueColumn.column_name, + [column.id]: barcodeValueColumn.column_name, }); break; } @@ -2225,14 +2223,14 @@ class BaseModelSqlv2 { qb.select( this.dbDriver.raw(`?? as ??`, [ selectQb.builder, - sanitize(column.title), + sanitize(column.id), ]), ); } catch (e) { console.log(e); // return dummy select qb.select( - this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.title)]), + this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.id)]), ); } } @@ -2249,13 +2247,13 @@ class BaseModelSqlv2 { alias, columnOptions: (await column.getColOptions()) as RollupColumn, }) - ).builder.as(sanitize(column.title)), + ).builder.as(sanitize(column.id)), ); break; default: if (this.isPg) { if (column.dt === 'bytea') { - res[sanitize(column.title || column.column_name)] = + res[sanitize(column.id || column.column_name)] = this.dbDriver.raw( `encode(??.??, '${ column.meta?.format === 'hex' ? 'hex' : 'escape' @@ -2266,7 +2264,7 @@ class BaseModelSqlv2 { } } - res[sanitize(column.title || column.column_name)] = sanitize( + res[sanitize(column.id || column.column_name)] = sanitize( `${alias || this.tnPath}.${column.column_name}`, ); break; @@ -2315,9 +2313,9 @@ class BaseModelSqlv2 { const query = this.dbDriver(this.tnPath).insert(insertObj); if ((this.isPg || this.isMssql) && this.model.primaryKey) { query.returning( - `${this.model.primaryKey.column_name} as ${this.model.primaryKey.title}`, + `${this.model.primaryKey.column_name} as ${this.model.primaryKey.id}`, ); - response = await this.execAndParse(query); + response = await this.execAndParse(query, null, { raw: true }); } const ai = this.model.columns.find((c) => c.ai); @@ -2329,7 +2327,7 @@ class BaseModelSqlv2 { if (ag) { if (!response) await this.execAndParse(query); response = await this.readByPk( - data[ag.title], + insertObj[ag.column_name], false, {}, { ignoreView: true, getHiddenColumn: true }, @@ -2380,8 +2378,8 @@ class BaseModelSqlv2 { } } else if (ai) { const id = Array.isArray(response) - ? response?.[0]?.[ai.title] - : response?.[ai.title]; + ? response?.[0]?.[ai.id] + : response?.[ai.id]; response = await this.readByPk( id, false, @@ -2677,9 +2675,9 @@ class BaseModelSqlv2 { if (this.isPg || this.isMssql) { query.returning( - `${this.model.primaryKey.column_name} as ${this.model.primaryKey.title}`, + `${this.model.primaryKey.column_name} as ${this.model.primaryKey.id}`, ); - response = await this.execAndParse(query); + response = await this.execAndParse(query, null, { raw: true }); } const ai = this.model.columns.find((c) => c.ai); @@ -2691,7 +2689,7 @@ class BaseModelSqlv2 { if (ag) { if (!response) await this.execAndParse(query); response = await this.readByPk( - data[ag.title], + insertObj[ag.column_name], false, {}, { ignoreView: true, getHiddenColumn: true }, @@ -2749,8 +2747,8 @@ class BaseModelSqlv2 { } } else if (ai) { rowId = Array.isArray(response) - ? response?.[0]?.[ai.title] - : response?.[ai.title]; + ? response?.[0]?.[ai.id] + : response?.[ai.id]; } await Promise.all(postInsertOps.map((f) => f(rowId))); @@ -4223,11 +4221,11 @@ class BaseModelSqlv2 { const groupedResult = result.reduce>( (aggObj, row) => { - if (!aggObj.has(row[column.title])) { - aggObj.set(row[column.title], []); + if (!aggObj.has(row[column.id])) { + aggObj.set(row[column.id], []); } - aggObj.get(row[column.title]).push(row); + aggObj.get(row[column.id]).push(row); return aggObj; }, @@ -4324,11 +4322,13 @@ class BaseModelSqlv2 { options: { skipDateConversion?: boolean; skipAttachmentConversion?: boolean; + skipSubstitutingColumnIds?: boolean; raw?: boolean; // alias for skipDateConversion and skipAttachmentConversion first?: boolean; } = { skipDateConversion: false, skipAttachmentConversion: false, + skipSubstitutingColumnIds: false, raw: false, first: false, }, @@ -4336,6 +4336,7 @@ class BaseModelSqlv2 { if (options.raw) { options.skipDateConversion = true; options.skipAttachmentConversion = true; + options.skipSubstitutingColumnIds = true; } if (options.first && typeof qb !== 'string') { @@ -4368,6 +4369,10 @@ class BaseModelSqlv2 { data = this.convertDateFormat(data, childTable); } + if (!options.skipSubstitutingColumnIds) { + data = this.substituteColumnIdsWithColumnTitles(data, childTable); + } + if (options.first) { return data?.[0]; } @@ -4375,6 +4380,35 @@ class BaseModelSqlv2 { return data; } + protected substituteColumnIdsWithColumnTitles( + data: Record[], + childTable?: Model, + ) { + const modelColumns = this.model?.columns.concat(childTable?.columns ?? []); + + if (!modelColumns || !data.length) { + return data; + } + + const idToAliasMap: Record = {}; + + modelColumns.forEach((col) => { + idToAliasMap[col.id] = col.title; + }); + + data.forEach((item) => { + Object.entries(item).forEach(([key, value]) => { + const alias = idToAliasMap[key]; + if (alias) { + item[alias] = value; + delete item[key]; + } + }); + }); + + return data; + } + protected async _convertAttachmentType( attachmentColumns: Record[], d: Record, @@ -4383,12 +4417,12 @@ class BaseModelSqlv2 { if (d) { const promises = []; for (const col of attachmentColumns) { - if (d[col.title] && typeof d[col.title] === 'string') { - d[col.title] = JSON.parse(d[col.title]); + if (d[col.id] && typeof d[col.id] === 'string') { + d[col.id] = JSON.parse(d[col.id]); } - if (d[col.title]?.length) { - for (const attachment of d[col.title]) { + if (d[col.id]?.length) { + for (const attachment of d[col.id]) { // we expect array of array of attachments in case of lookup if (Array.isArray(attachment)) { for (const lookedUpAttachment of attachment) { @@ -4502,24 +4536,24 @@ class BaseModelSqlv2 { ) { if (!d) return d; for (const col of dateTimeColumns) { - if (!d[col.title]) continue; + if (!d[col.id]) continue; if (col.uidt === UITypes.Formula) { - if (!d[col.title] || typeof d[col.title] !== 'string') { + if (!d[col.id] || typeof d[col.id] !== 'string') { continue; } // remove milliseconds if (this.isMySQL) { - d[col.title] = d[col.title].replace(/\.000000/g, ''); + d[col.id] = d[col.id].replace(/\.000000/g, ''); } else if (this.isMssql) { - d[col.title] = d[col.title].replace(/\.0000000 \+00:00/g, ''); + d[col.id] = d[col.id].replace(/\.0000000 \+00:00/g, ''); } - if (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g.test(d[col.title])) { + if (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g.test(d[col.id])) { // convert ISO string (e.g. in MSSQL) to YYYY-MM-DD hh:mm:ssZ // e.g. 2023-05-18T05:30:00.000Z -> 2023-05-18 11:00:00+05:30 - d[col.title] = d[col.title].replace( + d[col.id] = d[col.id].replace( /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g, (d: string) => { if (!dayjs(d).isValid()) return d; @@ -4536,7 +4570,7 @@ class BaseModelSqlv2 { // convert all date time values to utc // the datetime is either YYYY-MM-DD hh:mm:ss (xcdb) // or YYYY-MM-DD hh:mm:ss+/-xx:yy (ext) - d[col.title] = d[col.title].replace( + d[col.id] = d[col.id].replace( /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?/g, (d: string) => { if (!dayjs(d).isValid()) { @@ -4576,15 +4610,15 @@ class BaseModelSqlv2 { if (this.isSqlite) { if (!col.cdf) { if ( - d[col.title].indexOf('-') === -1 && - d[col.title].indexOf('+') === -1 && - d[col.title].slice(-1) !== 'Z' + d[col.id].indexOf('-') === -1 && + d[col.id].indexOf('+') === -1 && + d[col.id].slice(-1) !== 'Z' ) { // if there is no timezone info, // we assume the input is on NocoDB server timezone // then we convert to UTC from server timezone // e.g. 2023-04-27 10:00:00 (IST) -> 2023-04-27 04:30:00+00:00 - d[col.title] = dayjs(d[col.title]) + d[col.id] = dayjs(d[col.id]) .tz(Intl.DateTimeFormat().resolvedOptions().timeZone) .utc() .format('YYYY-MM-DD HH:mm:ssZ'); @@ -4602,14 +4636,14 @@ class BaseModelSqlv2 { keepLocalTime = false; } - if (d[col.title] instanceof Date) { + if (d[col.id] instanceof Date) { // e.g. MSSQL // Wed May 10 2023 17:47:46 GMT+0800 (Hong Kong Standard Time) keepLocalTime = false; } // e.g. 01.01.2022 10:00:00+05:30 -> 2022-01-01 04:30:00+00:00 // e.g. 2023-05-09 11:41:49 -> 2023-05-09 11:41:49+00:00 - d[col.title] = dayjs(d[col.title]) + d[col.id] = dayjs(d[col.id]) // keep the local time .utc(keepLocalTime) // show the timezone even for Mysql @@ -5413,7 +5447,9 @@ export function _wherePk(primaryKeys: Column[], id: unknown | unknown[]) { if (id && typeof id === 'object' && !Array.isArray(id)) { // verify all pk columns are present in id object for (const pk of primaryKeys) { - if (pk.title in id) { + if (pk.id in id) { + where[pk.column_name] = id[pk.id]; + } else if (pk.title in id) { where[pk.column_name] = id[pk.title]; } else if (pk.column_name in id) { where[pk.column_name] = id[pk.column_name];