From 4f0f38d953fde3bb96db67d98dfdcea9819d6b72 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 21 Nov 2023 14:16:43 +0530 Subject: [PATCH 001/326] feat: remove signup link from signup page if invite only signup enabled --- .../nc-gui/composables/useGlobal/state.ts | 1 + .../nc-gui/composables/useGlobal/types.ts | 1 + packages/nc-gui/pages/signin.vue | 2 +- packages/nocodb/src/services/utils.service.ts | 26 ++++++++++++------- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/nc-gui/composables/useGlobal/state.ts b/packages/nc-gui/composables/useGlobal/state.ts index 4fab0a5c72..318dea2a43 100644 --- a/packages/nc-gui/composables/useGlobal/state.ts +++ b/packages/nc-gui/composables/useGlobal/state.ts @@ -109,6 +109,7 @@ export function useGlobalState(storageKey = 'nocodb-gui-v2'): State { automationLogLevel: 'OFF', disableEmailAuth: false, dashboardPath: '/dashboard', + inviteOnlySignup: false, }) /** reactive token payload */ diff --git a/packages/nc-gui/composables/useGlobal/types.ts b/packages/nc-gui/composables/useGlobal/types.ts index f7e640cd7c..9efbfacb45 100644 --- a/packages/nc-gui/composables/useGlobal/types.ts +++ b/packages/nc-gui/composables/useGlobal/types.ts @@ -34,6 +34,7 @@ export interface AppInfo { disableEmailAuth: boolean mainSubDomain?: string dashboardPath: string + inviteOnlySignup: boolean } export interface StoredState { diff --git a/packages/nc-gui/pages/signin.vue b/packages/nc-gui/pages/signin.vue index 31cc73eb4e..63d3b35d77 100644 --- a/packages/nc-gui/pages/signin.vue +++ b/packages/nc-gui/pages/signin.vue @@ -184,7 +184,7 @@ function navigateForgotPassword() { -
+
{{ $t('msg.info.signUp.dontHaveAccount') }} {{ $t('general.signUp') }}
diff --git a/packages/nocodb/src/services/utils.service.ts b/packages/nocodb/src/services/utils.service.ts index 9cd676aef3..3abeeb31e1 100644 --- a/packages/nocodb/src/services/utils.service.ts +++ b/packages/nocodb/src/services/utils.service.ts @@ -5,10 +5,10 @@ import { ViewTypes } from 'nocodb-sdk'; import { ConfigService } from '@nestjs/config'; import { useAgent } from 'request-filtering-agent'; import type { AppConfig } from '~/interface/config'; -import { NC_ATTACHMENT_FIELD_SIZE } from '~/constants'; +import {NC_APP_SETTINGS, NC_ATTACHMENT_FIELD_SIZE} from '~/constants'; import SqlMgrv2 from '~/db/sql-mgr/v2/SqlMgrv2'; import { NcError } from '~/helpers/catchError'; -import { Base, User } from '~/models'; +import {Base, Store, User} from '~/models'; import Noco from '~/Noco'; import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2'; import { MetaTable } from '~/utils/globals'; @@ -365,6 +365,13 @@ export class UtilsService { async appInfo(param: { req: { ncSiteUrl: string } }) { const baseHasAdmin = !(await User.isFirst()); + + let settings: { invite_only_signup?: boolean } = {}; + try { + settings = JSON.parse((await Store.get(NC_APP_SETTINGS))?.value); + } catch {} + + const oidcAuthEnabled = !!( process.env.NC_OIDC_ISSUER && process.env.NC_OIDC_AUTHORIZATION_URL && @@ -384,10 +391,10 @@ export class UtilsService { type: 'rest', env: process.env.NODE_ENV, googleAuthEnabled: !!( - process.env.NC_GOOGLE_CLIENT_ID && process.env.NC_GOOGLE_CLIENT_SECRET + process.env.NC_GOOGLE_CLIENT_ID && process.env.NC_GOOGLE_CLIENT_SECRET ), githubAuthEnabled: !!( - process.env.NC_GITHUB_CLIENT_ID && process.env.NC_GITHUB_CLIENT_SECRET + process.env.NC_GITHUB_CLIENT_ID && process.env.NC_GITHUB_CLIENT_SECRET ), oidcAuthEnabled, oidcProviderName, @@ -395,8 +402,8 @@ export class UtilsService { connectToExternalDB: !process.env.NC_CONNECT_TO_EXTERNAL_DB_DISABLED, version: packageVersion, defaultLimit: Math.max( - Math.min(defaultLimitConfig.limitDefault, defaultLimitConfig.limitMax), - defaultLimitConfig.limitMin, + Math.min(defaultLimitConfig.limitDefault, defaultLimitConfig.limitMax), + defaultLimitConfig.limitMin, ), defaultGroupByLimit: defaultGroupByLimitConfig, timezone: defaultConnectionConfig.timezone, @@ -413,9 +420,10 @@ export class UtilsService { disableEmailAuth: this.configService.get('auth.disableEmailAuth', { infer: true, }), - mainSubDomain: this.configService.get('mainSubDomain', { infer: true }), - dashboardPath: this.configService.get('dashboardPath', { infer: true }), - }; + mainSubDomain: this.configService.get('mainSubDomain', {infer: true}), + dashboardPath: this.configService.get('dashboardPath', {infer: true}), + inviteOnlySignup: settings.invite_only_signup, + } return result; } From d9b03d3526c1ed617a56874ec74d13d4c2b41629 Mon Sep 17 00:00:00 2001 From: mertmit Date: Wed, 22 Nov 2023 11:40:44 +0000 Subject: [PATCH 002/326] feat: use id instead of alias on queries --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 2 +- packages/nocodb/src/db/BaseModelSqlv2.ts | 174 +++++++++++------- 2 files changed, 106 insertions(+), 70 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 5a28417d84..a5a15ed532 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -93,7 +93,7 @@ export function substituteColumnIdWithAliasInFormula( c.column_name === colNameOrId || c.title === colNameOrId ); - pt.name = column?.title || ptRaw?.name || pt?.name; + pt.name = column?.id || ptRaw?.name || pt?.name; } else if (pt.type === 'BinaryExpression') { substituteId(pt.left, ptRaw?.left); substituteId(pt.right, ptRaw?.right); diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index a60d09856d..089d5fc586 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -167,7 +167,9 @@ class BaseModelSqlv2 { let data; try { - data = await this.execAndParse(qb, null, { first: true }); + data = await this.execAndParse(qb, null, { + first: true, + }); } catch (e) { if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) throw e; @@ -587,9 +589,9 @@ class BaseModelSqlv2 { knex: this.dbDriver, columnOptions: (await column.getColOptions()) as RollupColumn, }) - ).builder.as(sanitize(column.title)), + ).builder.as(sanitize(column.id)), ); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); break; case UITypes.Formula: { @@ -601,18 +603,18 @@ class BaseModelSqlv2 { selectQb = this.dbDriver.raw(`?? as ??`, [ _selectQb.builder, - sanitize(column.title), + sanitize(column.id), ]); } catch (e) { console.log(e); // return dummy select selectQb = this.dbDriver.raw(`'ERR' as ??`, [ - sanitize(column.title), + sanitize(column.id), ]); } selectors.push(selectQb); - groupBySelectors.push(column.title); + groupBySelectors.push(column.id); } break; case UITypes.Lookup: @@ -628,18 +630,18 @@ class BaseModelSqlv2 { const selectQb = this.dbDriver.raw(`?? as ??`, [ this.dbDriver.raw(_selectQb.builder).wrap('(', ')'), - sanitize(column.title), + sanitize(column.id), ]); selectors.push(selectQb); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); } break; default: selectors.push( - this.dbDriver.raw('?? as ??', [column.column_name, column.title]), + this.dbDriver.raw('?? as ??', [column.column_name, column.id]), ); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); break; } }), @@ -704,7 +706,7 @@ class BaseModelSqlv2 { } qb.orderBy( - groupByColumns[sort.fk_column_id].title, + groupByColumns[sort.fk_column_id].id, sort.direction, sort.direction === 'desc' ? 'LAST' : 'FIRST', ); @@ -772,9 +774,9 @@ class BaseModelSqlv2 { columnOptions: (await column.getColOptions()) as RollupColumn, }) - ).builder.as(sanitize(column.title)), + ).builder.as(sanitize(column.id)), ); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); break; case UITypes.Formula: { let selectQb; @@ -785,18 +787,18 @@ class BaseModelSqlv2 { selectQb = this.dbDriver.raw(`?? as ??`, [ _selectQb.builder, - sanitize(column.title), + sanitize(column.id), ]); } catch (e) { console.log(e); // return dummy select selectQb = this.dbDriver.raw(`'ERR' as ??`, [ - sanitize(column.title), + sanitize(column.id), ]); } selectors.push(selectQb); - groupBySelectors.push(column.title); + groupBySelectors.push(column.id); break; } case UITypes.Lookup: @@ -812,21 +814,18 @@ class BaseModelSqlv2 { const selectQb = this.dbDriver.raw(`?? as ??`, [ this.dbDriver.raw(_selectQb.builder).wrap('(', ')'), - sanitize(column.title), + sanitize(column.id), ]); selectors.push(selectQb); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); } break; default: selectors.push( - this.dbDriver.raw('?? as ??', [ - column.column_name, - column.title, - ]), + this.dbDriver.raw('?? as ??', [column.column_name, column.id]), ); - groupBySelectors.push(sanitize(column.title)); + groupBySelectors.push(sanitize(column.id)); break; } }), @@ -2099,11 +2098,10 @@ class BaseModelSqlv2 { // the value 2023-01-01 10:00:00 (UTC) would display as 2023-01-01 18:00:00 (UTC+8) // our existing logic is based on UTC, during the query, we need to take the UTC value // hence, we use CONVERT_TZ to convert back to UTC value - res[sanitize(column.title || column.column_name)] = - this.dbDriver.raw( - `CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00')`, - [`${sanitize(alias || this.tnPath)}.${column.column_name}`], - ); + res[sanitize(column.id || column.column_name)] = this.dbDriver.raw( + `CONVERT_TZ(??, @@GLOBAL.time_zone, '+00:00')`, + [`${sanitize(alias || this.tnPath)}.${column.column_name}`], + ); break; } else if (this.isPg) { // if there is no timezone info, @@ -2113,7 +2111,7 @@ class BaseModelSqlv2 { column.dt !== 'timestamp with time zone' && column.dt !== 'timestamptz' ) { - res[sanitize(column.title || column.column_name)] = this.dbDriver + res[sanitize(column.id || column.column_name)] = this.dbDriver .raw( `?? AT TIME ZONE CURRENT_SETTING('timezone') AT TIME ZONE 'UTC'`, [`${sanitize(alias || this.tnPath)}.${column.column_name}`], @@ -2126,7 +2124,7 @@ class BaseModelSqlv2 { // convert to database timezone, // then convert to UTC if (column.dt !== 'datetimeoffset') { - res[sanitize(column.title || column.column_name)] = + res[sanitize(column.id || column.column_name)] = this.dbDriver.raw( `CONVERT(DATETIMEOFFSET, ?? AT TIME ZONE 'UTC')`, [`${sanitize(alias || this.tnPath)}.${column.column_name}`], @@ -2134,7 +2132,7 @@ class BaseModelSqlv2 { break; } } - res[sanitize(column.title || column.column_name)] = sanitize( + res[sanitize(column.id || column.column_name)] = sanitize( `${alias || this.tnPath}.${column.column_name}`, ); break; @@ -2197,7 +2195,7 @@ class BaseModelSqlv2 { aliasToColumnBuilder, ); qb.select({ - [column.title]: selectQb.builder, + [column.id]: selectQb.builder, }); } catch { continue; @@ -2205,7 +2203,7 @@ class BaseModelSqlv2 { break; default: { qb.select({ - [column.title]: barcodeValueColumn.column_name, + [column.id]: barcodeValueColumn.column_name, }); break; } @@ -2225,14 +2223,14 @@ class BaseModelSqlv2 { qb.select( this.dbDriver.raw(`?? as ??`, [ selectQb.builder, - sanitize(column.title), + sanitize(column.id), ]), ); } catch (e) { console.log(e); // return dummy select qb.select( - this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.title)]), + this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.id)]), ); } } @@ -2249,13 +2247,13 @@ class BaseModelSqlv2 { alias, columnOptions: (await column.getColOptions()) as RollupColumn, }) - ).builder.as(sanitize(column.title)), + ).builder.as(sanitize(column.id)), ); break; default: if (this.isPg) { if (column.dt === 'bytea') { - res[sanitize(column.title || column.column_name)] = + res[sanitize(column.id || column.column_name)] = this.dbDriver.raw( `encode(??.??, '${ column.meta?.format === 'hex' ? 'hex' : 'escape' @@ -2266,7 +2264,7 @@ class BaseModelSqlv2 { } } - res[sanitize(column.title || column.column_name)] = sanitize( + res[sanitize(column.id || column.column_name)] = sanitize( `${alias || this.tnPath}.${column.column_name}`, ); break; @@ -2315,9 +2313,9 @@ class BaseModelSqlv2 { const query = this.dbDriver(this.tnPath).insert(insertObj); if ((this.isPg || this.isMssql) && this.model.primaryKey) { query.returning( - `${this.model.primaryKey.column_name} as ${this.model.primaryKey.title}`, + `${this.model.primaryKey.column_name} as ${this.model.primaryKey.id}`, ); - response = await this.execAndParse(query); + response = await this.execAndParse(query, null, { raw: true }); } const ai = this.model.columns.find((c) => c.ai); @@ -2329,7 +2327,7 @@ class BaseModelSqlv2 { if (ag) { if (!response) await this.execAndParse(query); response = await this.readByPk( - data[ag.title], + insertObj[ag.column_name], false, {}, { ignoreView: true, getHiddenColumn: true }, @@ -2380,8 +2378,8 @@ class BaseModelSqlv2 { } } else if (ai) { const id = Array.isArray(response) - ? response?.[0]?.[ai.title] - : response?.[ai.title]; + ? response?.[0]?.[ai.id] + : response?.[ai.id]; response = await this.readByPk( id, false, @@ -2677,9 +2675,9 @@ class BaseModelSqlv2 { if (this.isPg || this.isMssql) { query.returning( - `${this.model.primaryKey.column_name} as ${this.model.primaryKey.title}`, + `${this.model.primaryKey.column_name} as ${this.model.primaryKey.id}`, ); - response = await this.execAndParse(query); + response = await this.execAndParse(query, null, { raw: true }); } const ai = this.model.columns.find((c) => c.ai); @@ -2691,7 +2689,7 @@ class BaseModelSqlv2 { if (ag) { if (!response) await this.execAndParse(query); response = await this.readByPk( - data[ag.title], + insertObj[ag.column_name], false, {}, { ignoreView: true, getHiddenColumn: true }, @@ -2749,8 +2747,8 @@ class BaseModelSqlv2 { } } else if (ai) { rowId = Array.isArray(response) - ? response?.[0]?.[ai.title] - : response?.[ai.title]; + ? response?.[0]?.[ai.id] + : response?.[ai.id]; } await Promise.all(postInsertOps.map((f) => f(rowId))); @@ -4223,11 +4221,11 @@ class BaseModelSqlv2 { const groupedResult = result.reduce>( (aggObj, row) => { - if (!aggObj.has(row[column.title])) { - aggObj.set(row[column.title], []); + if (!aggObj.has(row[column.id])) { + aggObj.set(row[column.id], []); } - aggObj.get(row[column.title]).push(row); + aggObj.get(row[column.id]).push(row); return aggObj; }, @@ -4324,11 +4322,13 @@ class BaseModelSqlv2 { options: { skipDateConversion?: boolean; skipAttachmentConversion?: boolean; + skipSubstitutingColumnIds?: boolean; raw?: boolean; // alias for skipDateConversion and skipAttachmentConversion first?: boolean; } = { skipDateConversion: false, skipAttachmentConversion: false, + skipSubstitutingColumnIds: false, raw: false, first: false, }, @@ -4336,6 +4336,7 @@ class BaseModelSqlv2 { if (options.raw) { options.skipDateConversion = true; options.skipAttachmentConversion = true; + options.skipSubstitutingColumnIds = true; } if (options.first && typeof qb !== 'string') { @@ -4368,6 +4369,10 @@ class BaseModelSqlv2 { data = this.convertDateFormat(data, childTable); } + if (!options.skipSubstitutingColumnIds) { + data = this.substituteColumnIdsWithColumnTitles(data, childTable); + } + if (options.first) { return data?.[0]; } @@ -4375,6 +4380,35 @@ class BaseModelSqlv2 { return data; } + protected substituteColumnIdsWithColumnTitles( + data: Record[], + childTable?: Model, + ) { + const modelColumns = this.model?.columns.concat(childTable?.columns ?? []); + + if (!modelColumns || !data.length) { + return data; + } + + const idToAliasMap: Record = {}; + + modelColumns.forEach((col) => { + idToAliasMap[col.id] = col.title; + }); + + data.forEach((item) => { + Object.entries(item).forEach(([key, value]) => { + const alias = idToAliasMap[key]; + if (alias) { + item[alias] = value; + delete item[key]; + } + }); + }); + + return data; + } + protected async _convertAttachmentType( attachmentColumns: Record[], d: Record, @@ -4383,12 +4417,12 @@ class BaseModelSqlv2 { if (d) { const promises = []; for (const col of attachmentColumns) { - if (d[col.title] && typeof d[col.title] === 'string') { - d[col.title] = JSON.parse(d[col.title]); + if (d[col.id] && typeof d[col.id] === 'string') { + d[col.id] = JSON.parse(d[col.id]); } - if (d[col.title]?.length) { - for (const attachment of d[col.title]) { + if (d[col.id]?.length) { + for (const attachment of d[col.id]) { // we expect array of array of attachments in case of lookup if (Array.isArray(attachment)) { for (const lookedUpAttachment of attachment) { @@ -4502,24 +4536,24 @@ class BaseModelSqlv2 { ) { if (!d) return d; for (const col of dateTimeColumns) { - if (!d[col.title]) continue; + if (!d[col.id]) continue; if (col.uidt === UITypes.Formula) { - if (!d[col.title] || typeof d[col.title] !== 'string') { + if (!d[col.id] || typeof d[col.id] !== 'string') { continue; } // remove milliseconds if (this.isMySQL) { - d[col.title] = d[col.title].replace(/\.000000/g, ''); + d[col.id] = d[col.id].replace(/\.000000/g, ''); } else if (this.isMssql) { - d[col.title] = d[col.title].replace(/\.0000000 \+00:00/g, ''); + d[col.id] = d[col.id].replace(/\.0000000 \+00:00/g, ''); } - if (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g.test(d[col.title])) { + if (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g.test(d[col.id])) { // convert ISO string (e.g. in MSSQL) to YYYY-MM-DD hh:mm:ssZ // e.g. 2023-05-18T05:30:00.000Z -> 2023-05-18 11:00:00+05:30 - d[col.title] = d[col.title].replace( + d[col.id] = d[col.id].replace( /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g, (d: string) => { if (!dayjs(d).isValid()) return d; @@ -4536,7 +4570,7 @@ class BaseModelSqlv2 { // convert all date time values to utc // the datetime is either YYYY-MM-DD hh:mm:ss (xcdb) // or YYYY-MM-DD hh:mm:ss+/-xx:yy (ext) - d[col.title] = d[col.title].replace( + d[col.id] = d[col.id].replace( /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?/g, (d: string) => { if (!dayjs(d).isValid()) { @@ -4576,15 +4610,15 @@ class BaseModelSqlv2 { if (this.isSqlite) { if (!col.cdf) { if ( - d[col.title].indexOf('-') === -1 && - d[col.title].indexOf('+') === -1 && - d[col.title].slice(-1) !== 'Z' + d[col.id].indexOf('-') === -1 && + d[col.id].indexOf('+') === -1 && + d[col.id].slice(-1) !== 'Z' ) { // if there is no timezone info, // we assume the input is on NocoDB server timezone // then we convert to UTC from server timezone // e.g. 2023-04-27 10:00:00 (IST) -> 2023-04-27 04:30:00+00:00 - d[col.title] = dayjs(d[col.title]) + d[col.id] = dayjs(d[col.id]) .tz(Intl.DateTimeFormat().resolvedOptions().timeZone) .utc() .format('YYYY-MM-DD HH:mm:ssZ'); @@ -4602,14 +4636,14 @@ class BaseModelSqlv2 { keepLocalTime = false; } - if (d[col.title] instanceof Date) { + if (d[col.id] instanceof Date) { // e.g. MSSQL // Wed May 10 2023 17:47:46 GMT+0800 (Hong Kong Standard Time) keepLocalTime = false; } // e.g. 01.01.2022 10:00:00+05:30 -> 2022-01-01 04:30:00+00:00 // e.g. 2023-05-09 11:41:49 -> 2023-05-09 11:41:49+00:00 - d[col.title] = dayjs(d[col.title]) + d[col.id] = dayjs(d[col.id]) // keep the local time .utc(keepLocalTime) // show the timezone even for Mysql @@ -5413,7 +5447,9 @@ export function _wherePk(primaryKeys: Column[], id: unknown | unknown[]) { if (id && typeof id === 'object' && !Array.isArray(id)) { // verify all pk columns are present in id object for (const pk of primaryKeys) { - if (pk.title in id) { + if (pk.id in id) { + where[pk.column_name] = id[pk.id]; + } else if (pk.title in id) { where[pk.column_name] = id[pk.title]; } else if (pk.column_name in id) { where[pk.column_name] = id[pk.column_name]; From 7f01d0ccabd68ff37f043d89f7985d5e5ae9e334 Mon Sep 17 00:00:00 2001 From: mertmit Date: Wed, 22 Nov 2023 11:40:44 +0000 Subject: [PATCH 003/326] fix: bt substitution --- packages/nocodb/src/db/BaseModelSqlv2.ts | 47 ++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 089d5fc586..10e984c5eb 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -1745,7 +1745,7 @@ class BaseModelSqlv2 { applyPaginate(qb, rest); const proto = await parentModel.getProto(); - const data = await this.execAndParse(qb, childTable); + const data = await this.execAndParse(qb, parentTable); return data.map((c) => { c.__proto__ = proto; @@ -4370,7 +4370,7 @@ class BaseModelSqlv2 { } if (!options.skipSubstitutingColumnIds) { - data = this.substituteColumnIdsWithColumnTitles(data, childTable); + data = await this.substituteColumnIdsWithColumnTitles(data, childTable); } if (options.first) { @@ -4380,7 +4380,7 @@ class BaseModelSqlv2 { return data; } - protected substituteColumnIdsWithColumnTitles( + protected async substituteColumnIdsWithColumnTitles( data: Record[], childTable?: Model, ) { @@ -4391,16 +4391,55 @@ class BaseModelSqlv2 { } const idToAliasMap: Record = {}; + const idToAliasPromiseMap: Record> = {}; + const btMap: Record = {}; modelColumns.forEach((col) => { idToAliasMap[col.id] = col.title; + if (col.colOptions?.type === 'bt') { + btMap[col.id] = true; + const btData = Object.values(data).find( + (d) => d[col.id] && Object.keys(d[col.id]), + ); + if (btData) { + for (const k of Object.keys(btData[col.id])) { + const btAlias = idToAliasMap[k]; + if (!btAlias) { + idToAliasPromiseMap[k] = Column.get({ colId: k }).then((col) => { + return col.title; + }); + } + } + } + } else { + btMap[col.id] = false; + } }); + for (const k of Object.keys(idToAliasPromiseMap)) { + idToAliasMap[k] = await idToAliasPromiseMap[k]; + } + data.forEach((item) => { Object.entries(item).forEach(([key, value]) => { const alias = idToAliasMap[key]; if (alias) { - item[alias] = value; + if (btMap[key]) { + if (value) { + const tempObj = {}; + Object.entries(value).forEach(([k, v]) => { + const btAlias = idToAliasMap[k]; + if (btAlias) { + tempObj[btAlias] = v; + } + }); + item[alias] = tempObj; + } else { + item[alias] = value; + } + } else { + item[alias] = value; + } delete item[key]; } }); From 88720651766006221c020db439bafc4ff720b246 Mon Sep 17 00:00:00 2001 From: mertmit Date: Wed, 22 Nov 2023 11:40:44 +0000 Subject: [PATCH 004/326] fix: groupedList --- packages/nocodb/src/db/BaseModelSqlv2.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 10e984c5eb..727dceb90f 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -4221,11 +4221,11 @@ class BaseModelSqlv2 { const groupedResult = result.reduce>( (aggObj, row) => { - if (!aggObj.has(row[column.id])) { - aggObj.set(row[column.id], []); + if (!aggObj.has(row[column.title])) { + aggObj.set(row[column.title], []); } - aggObj.get(row[column.id]).push(row); + aggObj.get(row[column.title]).push(row); return aggObj; }, From df2f175d788e7083ee79bb81269674587ce78041 Mon Sep 17 00:00:00 2001 From: mertmit Date: Wed, 22 Nov 2023 11:40:45 +0000 Subject: [PATCH 005/326] fix: btRead --- packages/nocodb/src/db/BaseModelSqlv2.ts | 28 ++++++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 727dceb90f..fd658ff7e5 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -4804,7 +4804,9 @@ class BaseModelSqlv2 { if (parentTable.primaryKey.column_name !== parentColumn.column_name) childRowsQb.select(parentTable.primaryKey.column_name); - const childRows = await childRowsQb; + const childRows = await this.execAndParse(childRowsQb, null, { + raw: true, + }); if (childRows.length !== childIds.length) { const missingIds = childIds.filter( @@ -4863,7 +4865,9 @@ class BaseModelSqlv2 { ); } - const childRows = await childRowsQb; + const childRows = await this.execAndParse(childRowsQb, null, { + raw: true, + }); if (childRows.length !== childIds.length) { const missingIds = childIds.filter( @@ -4915,7 +4919,10 @@ class BaseModelSqlv2 { .where(_wherePk(parentTable.primaryKeys, childIds[0])) .first(); - const childRow = await childRowsQb; + const childRow = await this.execAndParse(childRowsQb, null, { + first: true, + raw: true, + }); if (!childRow) { NcError.unprocessableEntity( @@ -5034,7 +5041,9 @@ class BaseModelSqlv2 { if (parentTable.primaryKey.column_name !== parentColumn.column_name) childRowsQb.select(parentTable.primaryKey.column_name); - const childRows = await childRowsQb; + const childRows = await this.execAndParse(childRowsQb, null, { + raw: true, + }); if (childRows.length !== childIds.length) { const missingIds = childIds.filter( @@ -5080,7 +5089,9 @@ class BaseModelSqlv2 { .select(childTable.primaryKey.column_name) .whereIn(childTable.primaryKey.column_name, childIds); - const childRows = await childRowsQb; + const childRows = await this.execAndParse(childRowsQb, null, { + raw: true, + }); if (childRows.length !== childIds.length) { const missingIds = childIds.filter( @@ -5136,7 +5147,10 @@ class BaseModelSqlv2 { .where(_wherePk(parentTable.primaryKeys, childIds[0])) .first(); - const childRow = await childRowsQb; + const childRow = await this.execAndParse(childRowsQb, null, { + first: true, + raw: true, + }); if (!childRow) { NcError.unprocessableEntity( @@ -5228,7 +5242,7 @@ class BaseModelSqlv2 { await parentModel.selectObject({ qb, fieldsSet: args.fieldSet }); - const parent = await this.execAndParse(qb, childTable, { + const parent = await this.execAndParse(qb, parentTable, { first: true, }); From 98581b4735b2efe31194449ec7e6b9c7bb380f47 Mon Sep 17 00:00:00 2001 From: mertmit Date: Wed, 22 Nov 2023 11:40:45 +0000 Subject: [PATCH 006/326] fix: qr and barcode columns --- packages/nocodb/src/db/BaseModelSqlv2.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index fd658ff7e5..371d8c5e8d 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -570,6 +570,7 @@ class BaseModelSqlv2 { .getColOptions() .then((col) => col.getValueColumn())), title: column.title, + id: column.id, }); groupByColumns[column.id] = column; @@ -753,6 +754,7 @@ class BaseModelSqlv2 { .getColOptions() .then((col) => col.getValueColumn())), title: column.title, + id: column.id, }); switch (column.uidt) { From 4ad4ac2cd34da66e04cf176e012fbb6d1548daaa Mon Sep 17 00:00:00 2001 From: mertmit Date: Wed, 22 Nov 2023 11:40:45 +0000 Subject: [PATCH 007/326] fix: grouped data list count --- packages/nocodb/src/db/BaseModelSqlv2.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 371d8c5e8d..14aebe9847 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -4312,7 +4312,8 @@ class BaseModelSqlv2 { await this.selectObject({ qb, - columns: [new Column({ ...column, title: 'key' })], + // replace id with 'key' as we select as id + columns: [new Column({ ...column, title: 'key', id: 'key' })], }); return await this.execAndParse(qb); From 89a8c053d389f7d68aa70630ce18cb559babefaa Mon Sep 17 00:00:00 2001 From: mertmit Date: Wed, 22 Nov 2023 11:40:45 +0000 Subject: [PATCH 008/326] fix: use raw for update query --- packages/nocodb/src/db/BaseModelSqlv2.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 14aebe9847..d3e385f9ec 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2563,7 +2563,7 @@ class BaseModelSqlv2 { .update(updateObj) .where(await this._wherePk(id)); - await this.execAndParse(query); + await this.execAndParse(query, null, { raw: true }); // const newData = await this.readByPk(id, false, {}, { ignoreView: true , getHiddenColumn: true}); From 69e6412f374c57f1aba488863acaefde038cbaaa Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 22 Nov 2023 12:04:42 +0530 Subject: [PATCH 009/326] fix: rerender LTAR options when switching types LTAR <=> Links since it reset props --- packages/nc-gui/components/smartsheet/column/EditOrAdd.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/nc-gui/components/smartsheet/column/EditOrAdd.vue b/packages/nc-gui/components/smartsheet/column/EditOrAdd.vue index 9eb7a391a3..75f3625599 100644 --- a/packages/nc-gui/components/smartsheet/column/EditOrAdd.vue +++ b/packages/nc-gui/components/smartsheet/column/EditOrAdd.vue @@ -306,6 +306,7 @@ if (props.fromTableExplorer) { From 2f257b98d25f958104c6ac696e68b4c16001922d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 22 Nov 2023 12:26:23 +0530 Subject: [PATCH 010/326] feat: support Lookup sort --- packages/nocodb/src/db/sortV2.ts | 138 ++----------------------------- 1 file changed, 8 insertions(+), 130 deletions(-) diff --git a/packages/nocodb/src/db/sortV2.ts b/packages/nocodb/src/db/sortV2.ts index 0871c6c870..c4d20d6ebd 100644 --- a/packages/nocodb/src/db/sortV2.ts +++ b/packages/nocodb/src/db/sortV2.ts @@ -12,6 +12,7 @@ import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2'; import { sanitize } from '~/helpers/sqlSanitize'; import { Sort } from '~/models'; +import generateLookupSelectQuery from '~/db/generateLookupSelectQuery'; export default async function sortV2( baseModelSqlv2: BaseModelSqlv2, @@ -80,137 +81,14 @@ export default async function sortV2( { const rootAlias = alias; { - let aliasCount = 0, - selectQb; - const alias = `__nc_sort${aliasCount++}`; - const lookup = await column.getColOptions(); - { - const relationCol = await lookup.getRelationColumn(); - const relation = - await relationCol.getColOptions(); - if (relation.type !== RelationTypes.BELONGS_TO) return; - - const childColumn = await relation.getChildColumn(); - const parentColumn = await relation.getParentColumn(); - const childModel = await childColumn.getModel(); - await childModel.getColumns(); - const parentModel = await parentColumn.getModel(); - await parentModel.getColumns(); - - selectQb = knex( - `${baseModelSqlv2.getTnPath( - parentModel.table_name, - )} as ${alias}`, - ).where( - `${alias}.${parentColumn.column_name}`, - knex.raw(`??`, [ - `${ - rootAlias || baseModelSqlv2.getTnPath(childModel.table_name) - }.${childColumn.column_name}`, - ]), - ); - } - let lookupColumn = await lookup.getLookupColumn(); - let prevAlias = alias; - while (lookupColumn.uidt === UITypes.Lookup) { - const nestedAlias = `__nc_sort${aliasCount++}`; - const nestedLookup = - await lookupColumn.getColOptions(); - const relationCol = await nestedLookup.getRelationColumn(); - const relation = - await relationCol.getColOptions(); - // if any of the relation in nested lookup is - // not belongs to then ignore the sort option - if (relation.type !== 'bt') return; - - const childColumn = await relation.getChildColumn(); - const parentColumn = await relation.getParentColumn(); - const childModel = await childColumn.getModel(); - await childModel.getColumns(); - const parentModel = await parentColumn.getModel(); - await parentModel.getColumns(); - - selectQb.join( - `${baseModelSqlv2.getTnPath( - parentModel.table_name, - )} as ${nestedAlias}`, - `${nestedAlias}.${parentColumn.column_name}`, - `${prevAlias}.${childColumn.column_name}`, - ); - - lookupColumn = await nestedLookup.getLookupColumn(); - prevAlias = nestedAlias; - } - - switch (lookupColumn.uidt) { - case UITypes.Links: - case UITypes.Rollup: - { - const builder = ( - await genRollupSelectv2({ - baseModelSqlv2, - knex, - columnOptions: - (await lookupColumn.getColOptions()) as RollupColumn, - alias: prevAlias, - }) - ).builder; - selectQb.select(builder); - } - break; - case UITypes.LinkToAnotherRecord: - { - const nestedAlias = `__nc_sort${aliasCount++}`; - const relation = - await lookupColumn.getColOptions(); - if (relation.type !== 'bt') return; - - const colOptions = - (await column.getColOptions()) as LinkToAnotherRecordColumn; - const childColumn = await colOptions.getChildColumn(); - const parentColumn = await colOptions.getParentColumn(); - const childModel = await childColumn.getModel(); - await childModel.getColumns(); - const parentModel = await parentColumn.getModel(); - await parentModel.getColumns(); - - selectQb - .join( - `${baseModelSqlv2.getTnPath( - parentModel.table_name, - )} as ${nestedAlias}`, - `${nestedAlias}.${parentColumn.column_name}`, - `${prevAlias}.${childColumn.column_name}`, - ) - .select(parentModel?.displayValue?.column_name); - } - break; - case UITypes.Formula: - { - const builder = ( - await formulaQueryBuilderv2( - baseModelSqlv2, - ( - await column.getColOptions() - ).formula, - null, - model, - column, - ) - ).builder; - - selectQb.select(builder); - } - break; - default: - { - selectQb.select(`${prevAlias}.${lookupColumn.column_name}`); - } - - break; - } + const selectQb = await generateLookupSelectQuery({ + baseModelSqlv2, + column, + alias: rootAlias, + model, + }); - qb.orderBy(selectQb, sort.direction || 'asc', nulls); + qb.orderBy(selectQb?.builder, sort.direction || 'asc', nulls); } } break; From 8f05c30ddb0cbb888bb9de5d4cde64bbdbf68385 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 23 Nov 2023 11:40:53 +0530 Subject: [PATCH 011/326] feat: support LTAR sort --- packages/nocodb/src/db/sortV2.ts | 34 +------------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/packages/nocodb/src/db/sortV2.ts b/packages/nocodb/src/db/sortV2.ts index c4d20d6ebd..0cbcefae83 100644 --- a/packages/nocodb/src/db/sortV2.ts +++ b/packages/nocodb/src/db/sortV2.ts @@ -78,6 +78,7 @@ export default async function sortV2( } break; case UITypes.Lookup: + case UITypes.LinkToAnotherRecord: { const rootAlias = alias; { @@ -92,39 +93,6 @@ export default async function sortV2( } } break; - case UITypes.LinkToAnotherRecord: - { - const relation = - await column.getColOptions(); - if (relation.type !== 'bt') return; - - const colOptions = - (await column.getColOptions()) as LinkToAnotherRecordColumn; - const childColumn = await colOptions.getChildColumn(); - const parentColumn = await colOptions.getParentColumn(); - const childModel = await childColumn.getModel(); - await childModel.getColumns(); - const parentModel = await parentColumn.getModel(); - await parentModel.getColumns(); - - const selectQb = knex( - baseModelSqlv2.getTnPath(parentModel.table_name), - ) - .select(parentModel?.displayValue?.column_name) - .where( - `${baseModelSqlv2.getTnPath(parentModel.table_name)}.${ - parentColumn.column_name - }`, - knex.raw(`??`, [ - `${baseModelSqlv2.getTnPath(childModel.table_name)}.${ - childColumn.column_name - }`, - ]), - ); - - qb.orderBy(selectQb, sort.direction || 'asc', nulls); - } - break; case UITypes.SingleSelect: { const clientType = knex.clientType(); if (clientType === 'mysql' || clientType === 'mysql2') { From bf8020828f7b7052089e74e782ae655732503955 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 23 Nov 2023 12:09:15 +0530 Subject: [PATCH 012/326] chore: add comment --- .../src/db/generateLookupSelectQuery.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/db/generateLookupSelectQuery.ts b/packages/nocodb/src/db/generateLookupSelectQuery.ts index 758b5894db..3ea73fac94 100644 --- a/packages/nocodb/src/db/generateLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateLookupSelectQuery.ts @@ -28,6 +28,11 @@ export async function getDisplayValueOfRefTable( .then((cols) => cols.find((col) => col.pv)); } +// this function will generate the query for lookup column +// or for LTAR column and return the query builder +// query result will be aggregated json array string in case of Myssql and Postgres +// and string with separator in case of sqlite and mysql +// this function is used for sorting and grouping of lookup/LTAR column at the moment export default async function generateLookupSelectQuery({ column, baseModelSqlv2, @@ -392,8 +397,23 @@ export default async function generateLookupSelectQuery({ ) .from(selectQb.as(subQueryAlias)), }; + } else if (baseModelSqlv2.isMssql) { + // ref: https://stackoverflow.com/questions/13382856/sqlite3-join-group-concat-using-distinct-with-custom-separator + // selectQb.orderBy(`${lookupColumn.title}`, 'asc'); + return { + builder: knex + .select( + knex.raw(`STRING_AGG(??, ?)`, [ + lookupColumn.title, + LOOKUP_VAL_SEPARATOR, + ]), + ) + .from(selectQb.as(subQueryAlias)), + }; } - NcError.notImplemented('Database not supported Group by on Lookup'); + NcError.notImplemented( + 'Database not supported this operation on Lookup/LTAR', + ); } } From 0e3fe37362880f5407e208c65ab108af02cd27cc Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 23 Nov 2023 12:20:45 +0530 Subject: [PATCH 013/326] fix: qr code and bar code rendering in lookup re #6988 --- packages/nc-gui/components/virtual-cell/Lookup.vue | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/nc-gui/components/virtual-cell/Lookup.vue b/packages/nc-gui/components/virtual-cell/Lookup.vue index 6a2728fbef..4b0a81d755 100644 --- a/packages/nc-gui/components/virtual-cell/Lookup.vue +++ b/packages/nc-gui/components/virtual-cell/Lookup.vue @@ -110,9 +110,13 @@ const { showEditNonEditableFieldWarning, showClearNonEditableFieldWarning, activ >