From d422b729e0a5ec75510bc81dab7924d787de66eb Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:06 +0000 Subject: [PATCH 01/45] feat: lookup support - mm/hm lookup --- .../smartsheet/toolbar/GroupByMenu.vue | 12 +- .../src/db/generateBTLookupSelectQuery.ts | 11 +- .../src/db/generateMMLookupSelectQuery.ts | 204 ++++++++++++++++++ 3 files changed, 216 insertions(+), 11 deletions(-) create mode 100644 packages/nocodb/src/db/generateMMLookupSelectQuery.ts diff --git a/packages/nc-gui/components/smartsheet/toolbar/GroupByMenu.vue b/packages/nc-gui/components/smartsheet/toolbar/GroupByMenu.vue index 3febd97e8e..1b62c363bc 100644 --- a/packages/nc-gui/components/smartsheet/toolbar/GroupByMenu.vue +++ b/packages/nc-gui/components/smartsheet/toolbar/GroupByMenu.vue @@ -68,11 +68,11 @@ const fieldsToGroupBy = computed(() => { const fields = meta.value?.columns || [] return fields.filter((field) => { - if (!groupingUidt.includes(field.uidt as UITypes)) return false - - if (field.uidt === UITypes.Lookup) { - return btLookups.value.includes(field.id) - } + // if (!groupingUidt.includes(field.uidt as UITypes)) return false + // + // if (field.uidt === UITypes.Lookup) { + // return btLookups.value.includes(field.id) + // } return true }) @@ -243,7 +243,7 @@ onMounted(async () => { v-model="group.fk_column_id" class="caption nc-sort-field-select" :columns=" - fieldsToGroupBy.filter((f) => (f.id && !groupedByColumnIds.includes(f.id)) || f.id === group.fk_column_id) + fieldsToGroupBy " :allow-empty="true" @change="saveGroupBy" diff --git a/packages/nocodb/src/db/generateBTLookupSelectQuery.ts b/packages/nocodb/src/db/generateBTLookupSelectQuery.ts index 62835a1a44..5883fd0b56 100644 --- a/packages/nocodb/src/db/generateBTLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateBTLookupSelectQuery.ts @@ -38,9 +38,9 @@ export default async function generateBTLookupSelectQuery({ await relationCol.getColOptions(); // if not belongs to then throw error as we don't support - if (relation.type !== RelationTypes.BELONGS_TO) - NcError.badRequest('HasMany/ManyToMany lookup is not supported'); - + if (relation.type !== RelationTypes.BELONGS_TO) { + // NcError.badRequest('HasMany/ManyToMany lookup is not supported'); + } const childColumn = await relation.getChildColumn(); const parentColumn = await relation.getParentColumn(); const childModel = await childColumn.getModel(); @@ -70,8 +70,9 @@ export default async function generateBTLookupSelectQuery({ // if any of the relation in nested lookup is // not belongs to then throw error as we don't support - if (relation.type !== RelationTypes.BELONGS_TO) - NcError.badRequest('HasMany/ManyToMany lookup is not supported'); + if (relation.type !== RelationTypes.BELONGS_TO) { + // NcError.badRequest('HasMany/ManyToMany lookup is not supported'); + } const childColumn = await relation.getChildColumn(); const parentColumn = await relation.getParentColumn(); diff --git a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts new file mode 100644 index 0000000000..baf5a703ae --- /dev/null +++ b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts @@ -0,0 +1,204 @@ +import { RelationTypes, UITypes } from 'nocodb-sdk'; +import type LookupColumn from '../models/LookupColumn'; +import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; +import type { + Column, + FormulaColumn, + LinkToAnotherRecordColumn, + Model, + RollupColumn, +} from '~/models'; +import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; +import genRollupSelectv2 from '~/db/genRollupSelectv2'; +import { NcError } from '~/helpers/catchError'; + +export default async function generateBTLookupSelectQuery({ + column, + baseModelSqlv2, + alias, + model, +}: { + column: Column; + baseModelSqlv2: BaseModelSqlv2; + alias: string; + model: Model; +}): Promise { + const knex = baseModelSqlv2.dbDriver; + + const rootAlias = alias; + + { + let aliasCount = 0, + selectQb; + const alias = `__nc_lk_${aliasCount++}`; + const lookup = await column.getColOptions(); + { + const relationCol = await lookup.getRelationColumn(); + const relation = + await relationCol.getColOptions(); + + // if not belongs to then throw error as we don't support + if (relation.type !== RelationTypes.BELONGS_TO) { + // NcError.badRequest('HasMany/ManyToMany lookup is not supported'); + } + const childColumn = await relation.getChildColumn(); + const parentColumn = await relation.getParentColumn(); + const childModel = await childColumn.getModel(); + await childModel.getColumns(); + const parentModel = await parentColumn.getModel(); + await parentModel.getColumns(); + + `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`, + ).where( + selectQb = knex( + `${alias}.${parentColumn.column_name}`, + knex.raw(`??`, [ + `${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ + childColumn.column_name + }`, + ]), + ); + } + let lookupColumn = await lookup.getLookupColumn(); + let prevAlias = alias; + while (lookupColumn.uidt === UITypes.Lookup) { + const nestedAlias = `__nc_lk_nested_${aliasCount++}`; + const nestedLookup = await lookupColumn.getColOptions(); + const relationCol = await nestedLookup.getRelationColumn(); + const relation = + await relationCol.getColOptions(); + + // if any of the relation in nested lookup is + // not belongs to then throw error as we don't support + if (relation.type !== RelationTypes.BELONGS_TO) { + // NcError.badRequest('HasMany/ManyToMany lookup is not supported'); + } + + const childColumn = await relation.getChildColumn(); + const parentColumn = await relation.getParentColumn(); + const childModel = await childColumn.getModel(); + await childModel.getColumns(); + const parentModel = await parentColumn.getModel(); + await parentModel.getColumns(); + + selectQb.join( + `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${nestedAlias}`, + `${nestedAlias}.${parentColumn.column_name}`, + `${prevAlias}.${childColumn.column_name}`, + ); + + + + + + const mmModel = await relationColumnOption.getMMModel(); + const mmChildCol = await relationColumnOption.getMMChildColumn(); + const mmParentCol = await relationColumnOption.getMMParentColumn(); + + knex( + `${baseModelSqlv2.getTnPath( + parentModel?.table_name, + )} as ${refTableAlias}`, + ) + [columnOptions.rollup_function as string]?.( + knex.ref(`${refTableAlias}.${rollupColumn.column_name}`), + ) + .innerJoin( + baseModelSqlv2.getTnPath(mmModel.table_name), + knex.ref( + `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ + mmParentCol.column_name + }`, + ), + '=', + knex.ref(`${refTableAlias}.${parentCol.column_name}`), + ) + .where( + knex.ref( + `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ + mmChildCol.column_name + }`, + ), + '=', + knex.ref( + `${alias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ + childCol.column_name + }`, + ), + ) + + lookupColumn = await nestedLookup.getLookupColumn(); + prevAlias = nestedAlias; + } + + switch (lookupColumn.uidt) { + case UITypes.Links: + case UITypes.Rollup: + { + const builder = ( + await genRollupSelectv2({ + baseModelSqlv2, + knex, + columnOptions: + (await lookupColumn.getColOptions()) as RollupColumn, + alias: prevAlias, + }) + ).builder; + selectQb.select(builder); + } + break; + case UITypes.LinkToAnotherRecord: + { + const nestedAlias = `__nc_sort${aliasCount++}`; + const relation = + await lookupColumn.getColOptions(); + if (relation.type !== 'bt') return; + + const colOptions = + (await column.getColOptions()) as LinkToAnotherRecordColumn; + const childColumn = await colOptions.getChildColumn(); + const parentColumn = await colOptions.getParentColumn(); + const childModel = await childColumn.getModel(); + await childModel.getColumns(); + const parentModel = await parentColumn.getModel(); + await parentModel.getColumns(); + + selectQb + .join( + `${baseModelSqlv2.getTnPath( + parentModel.table_name, + )} as ${nestedAlias}`, + `${nestedAlias}.${parentColumn.column_name}`, + `${prevAlias}.${childColumn.column_name}`, + ) + .select(parentModel?.displayValue?.column_name); + } + break; + case UITypes.Formula: + { + const builder = ( + await formulaQueryBuilderv2( + baseModelSqlv2, + ( + await column.getColOptions() + ).formula, + null, + model, + column, + ) + ).builder; + + selectQb.select(builder); + } + break; + default: + { + selectQb.select(`${prevAlias}.${lookupColumn.column_name}`); + } + + break; + } + + return { builder: selectQb }; + } +} From 11d5c930c02e995d77c629f54d459f40fab368a2 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:10 +0000 Subject: [PATCH 02/45] feat: lookup support - mm/hm lookup - WIP --- .../src/db/generateMMLookupSelectQuery.ts | 108 ++++++++++-------- 1 file changed, 58 insertions(+), 50 deletions(-) diff --git a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts index baf5a703ae..767ec5e6e9 100644 --- a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts @@ -11,18 +11,23 @@ import type { import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2'; import { NcError } from '~/helpers/catchError'; +import {getAliasGenerator} from "~/utils"; export default async function generateBTLookupSelectQuery({ column, baseModelSqlv2, alias, model, + getAlias = getAliasGenerator('__lk_slt_') }: { column: Column; baseModelSqlv2: BaseModelSqlv2; alias: string; model: Model; + getAlias: ReturnType }): Promise { + + const knex = baseModelSqlv2.dbDriver; const rootAlias = alias; @@ -30,7 +35,7 @@ export default async function generateBTLookupSelectQuery({ { let aliasCount = 0, selectQb; - const alias = `__nc_lk_${aliasCount++}`; + const alias = getAlias(); const lookup = await column.getColOptions(); { const relationCol = await lookup.getRelationColumn(); @@ -62,7 +67,7 @@ export default async function generateBTLookupSelectQuery({ let lookupColumn = await lookup.getLookupColumn(); let prevAlias = alias; while (lookupColumn.uidt === UITypes.Lookup) { - const nestedAlias = `__nc_lk_nested_${aliasCount++}`; + const nestedAlias = getAlias(); const nestedLookup = await lookupColumn.getColOptions(); const relationCol = await nestedLookup.getRelationColumn(); const relation = @@ -70,62 +75,65 @@ export default async function generateBTLookupSelectQuery({ // if any of the relation in nested lookup is // not belongs to then throw error as we don't support - if (relation.type !== RelationTypes.BELONGS_TO) { - // NcError.badRequest('HasMany/ManyToMany lookup is not supported'); - } + if (relation.type === RelationTypes.BELONGS_TO) { - const childColumn = await relation.getChildColumn(); - const parentColumn = await relation.getParentColumn(); - const childModel = await childColumn.getModel(); - await childModel.getColumns(); - const parentModel = await parentColumn.getModel(); - await parentModel.getColumns(); + const childColumn = await relation.getChildColumn(); + const parentColumn = await relation.getParentColumn(); + const childModel = await childColumn.getModel(); + await childModel.getColumns(); + const parentModel = await parentColumn.getModel(); + await parentModel.getColumns(); - selectQb.join( - `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${nestedAlias}`, - `${nestedAlias}.${parentColumn.column_name}`, - `${prevAlias}.${childColumn.column_name}`, - ); + selectQb.join( + `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${nestedAlias}`, + `${nestedAlias}.${parentColumn.column_name}`, + `${prevAlias}.${childColumn.column_name}`, + ); + } else if (relation.type === RelationTypes.HAS_MANY) { + }else if (relation.type === RelationTypes.MANY_TO_MANY) { + const mmTableAlias = getAlias() + const mmModel = await relation.getMMModel(); + const mmChildCol = await relation.getMMChildColumn(); + const mmParentCol = await relation.getMMParentColumn(); + + knex( + `${baseModelSqlv2.getTnPath( + parentModel?.table_name, + )} as ${refTableAlias}`, + ) + [columnOptions.rollup_function as string]?.( + knex.ref(`${refTableAlias}.${rollupColumn.column_name}`), + ) + .innerJoin( + baseModelSqlv2.getTnPath(mmModel.table_name), + knex.ref( + `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ + mmParentCol.column_name + }`, + ), + '=', + knex.ref(`${refTableAlias}.${parentCol.column_name}`), + ) + .where( + knex.ref( + `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ + mmChildCol.column_name + }`, + ), + '=', + knex.ref( + `${alias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ + childCol.column_name + }`, + ), + ) + } - const mmModel = await relationColumnOption.getMMModel(); - const mmChildCol = await relationColumnOption.getMMChildColumn(); - const mmParentCol = await relationColumnOption.getMMParentColumn(); - knex( - `${baseModelSqlv2.getTnPath( - parentModel?.table_name, - )} as ${refTableAlias}`, - ) - [columnOptions.rollup_function as string]?.( - knex.ref(`${refTableAlias}.${rollupColumn.column_name}`), - ) - .innerJoin( - baseModelSqlv2.getTnPath(mmModel.table_name), - knex.ref( - `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ - mmParentCol.column_name - }`, - ), - '=', - knex.ref(`${refTableAlias}.${parentCol.column_name}`), - ) - .where( - knex.ref( - `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ - mmChildCol.column_name - }`, - ), - '=', - knex.ref( - `${alias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ - childCol.column_name - }`, - ), - ) lookupColumn = await nestedLookup.getLookupColumn(); prevAlias = nestedAlias; From 646aa1db5e8789992bb6ad0ca1dfc649c256dfdb Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:10 +0000 Subject: [PATCH 03/45] feat: lookup support - mm/hm lookup - WIP --- docker-compose/mysql/docker-compose.yml | 1 + packages/nocodb/src/db/BaseModelSqlv2.ts | 7 +- .../src/db/generateMMLookupSelectQuery.ts | 140 ++++++++++-------- 3 files changed, 83 insertions(+), 65 deletions(-) diff --git a/docker-compose/mysql/docker-compose.yml b/docker-compose/mysql/docker-compose.yml index 76e2e232cc..7ed99b7ee8 100644 --- a/docker-compose/mysql/docker-compose.yml +++ b/docker-compose/mysql/docker-compose.yml @@ -37,3 +37,4 @@ services: volumes: db_data: {} nc_data: {} + diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 9302691854..80f6fcfd78 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -64,6 +64,7 @@ import { } from '~/utils/globals'; import { extractProps } from '~/helpers/extractProps'; import { defaultLimitConfig } from '~/helpers/extractLimitAndOffset'; +import generateMMLookupSelectQuery from "~/db/generateMMLookupSelectQuery"; dayjs.extend(utc); @@ -600,7 +601,7 @@ class BaseModelSqlv2 { break; case UITypes.Lookup: { - const _selectQb = await generateBTLookupSelectQuery({ + const _selectQb = await generateMMLookupSelectQuery({ baseModelSqlv2: this, column, alias: null, @@ -695,6 +696,10 @@ class BaseModelSqlv2 { qb.groupBy(...groupBySelectors); applyPaginate(qb, rest); + + console.log('========') + console.log(qb.toQuery()) + console.log('========') return await this.execAndParse(qb); } diff --git a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts index 767ec5e6e9..0af42ea1ef 100644 --- a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts @@ -11,23 +11,21 @@ import type { import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2'; import { NcError } from '~/helpers/catchError'; -import {getAliasGenerator} from "~/utils"; +import { getAliasGenerator } from '~/utils'; -export default async function generateBTLookupSelectQuery({ +export default async function generateMMLookupSelectQuery({ column, baseModelSqlv2, alias, model, - getAlias = getAliasGenerator('__lk_slt_') + getAlias = getAliasGenerator('__lk_slt_'), }: { column: Column; baseModelSqlv2: BaseModelSqlv2; alias: string; model: Model; - getAlias: ReturnType + getAlias: ReturnType; }): Promise { - - const knex = baseModelSqlv2.dbDriver; const rootAlias = alias; @@ -53,9 +51,9 @@ export default async function generateBTLookupSelectQuery({ const parentModel = await parentColumn.getModel(); await parentModel.getColumns(); - `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`, - ).where( selectQb = knex( + `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`, + ).where( `${alias}.${parentColumn.column_name}`, knex.raw(`??`, [ `${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ @@ -76,65 +74,79 @@ export default async function generateBTLookupSelectQuery({ // if any of the relation in nested lookup is // not belongs to then throw error as we don't support if (relation.type === RelationTypes.BELONGS_TO) { - - const childColumn = await relation.getChildColumn(); - const parentColumn = await relation.getParentColumn(); - const childModel = await childColumn.getModel(); - await childModel.getColumns(); - const parentModel = await parentColumn.getModel(); - await parentModel.getColumns(); - - selectQb.join( - `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${nestedAlias}`, - `${nestedAlias}.${parentColumn.column_name}`, - `${prevAlias}.${childColumn.column_name}`, - ); - - } else if (relation.type === RelationTypes.HAS_MANY) { - - }else if (relation.type === RelationTypes.MANY_TO_MANY) { - const mmTableAlias = getAlias() - - const mmModel = await relation.getMMModel(); - const mmChildCol = await relation.getMMChildColumn(); - const mmParentCol = await relation.getMMParentColumn(); - - knex( - `${baseModelSqlv2.getTnPath( - parentModel?.table_name, - )} as ${refTableAlias}`, + const childColumn = await relation.getChildColumn(); + const parentColumn = await relation.getParentColumn(); + const childModel = await childColumn.getModel(); + await childModel.getColumns(); + const parentModel = await parentColumn.getModel(); + await parentModel.getColumns(); + + selectQb.join( + `${baseModelSqlv2.getTnPath( + parentModel.table_name, + )} as ${nestedAlias}`, + `${nestedAlias}.${parentColumn.column_name}`, + `${prevAlias}.${childColumn.column_name}`, + ); + } else if (relation.type === RelationTypes.HAS_MANY) { + const childColumn = await relation.getChildColumn(); + const parentColumn = await relation.getParentColumn(); + const childModel = await childColumn.getModel(); + await childModel.getColumns(); + const parentModel = await parentColumn.getModel(); + await parentModel.getColumns(); + + selectQb.join( + `${baseModelSqlv2.getTnPath( + childModel.table_name, + )} as ${nestedAlias}`, + `${nestedAlias}.${parentColumn.column_name}`, + `${prevAlias}.${childColumn.column_name}`, + ); + } else if (relation.type === RelationTypes.MANY_TO_MANY) { + const childColumn = await relation.getChildColumn(); + const parentColumn = await relation.getParentColumn(); + const childModel = await childColumn.getModel(); + await childModel.getColumns(); + const parentModel = await parentColumn.getModel(); + await parentModel.getColumns(); + + const mmTableAlias = getAlias(); + + const mmModel = await relation.getMMModel(); + const mmChildCol = await relation.getMMChildColumn(); + const mmParentCol = await relation.getMMParentColumn(); + + knex( + `${baseModelSqlv2.getTnPath( + parentModel?.table_name, + )} as ${nestedAlias}`, + ) + .innerJoin( + baseModelSqlv2.getTnPath(mmModel.table_name), + knex.ref( + `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ + mmParentCol.column_name + }`, + ), + '=', + knex.ref(`${nestedAlias}.${parentColumn.column_name}`), ) - [columnOptions.rollup_function as string]?.( - knex.ref(`${refTableAlias}.${rollupColumn.column_name}`), - ) - .innerJoin( - baseModelSqlv2.getTnPath(mmModel.table_name), - knex.ref( - `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ - mmParentCol.column_name - }`, - ), - '=', - knex.ref(`${refTableAlias}.${parentCol.column_name}`), - ) - .where( - knex.ref( - `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ - mmChildCol.column_name - }`, - ), - '=', - knex.ref( - `${alias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ - childCol.column_name - }`, - ), - ) + .where( + knex.ref( + `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ + mmChildCol.column_name + }`, + ), + '=', + knex.ref( + `${alias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ + childColumn.column_name + }`, + ), + ); } - - - lookupColumn = await nestedLookup.getLookupColumn(); prevAlias = nestedAlias; } From 86238145dabbbb9d2f36e591461046facd43d88f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:10 +0000 Subject: [PATCH 04/45] feat: group by - lookup support - mm/hm lookup - WIP --- packages/nocodb/src/db/BaseModelSqlv2.ts | 3 +- .../src/db/generateMMLookupSelectQuery.ts | 142 ++++++++++++++---- 2 files changed, 115 insertions(+), 30 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 80f6fcfd78..69a338a863 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -35,7 +35,6 @@ import type { SelectOption, } from '~/models'; import type { SortType } from 'nocodb-sdk'; -import generateBTLookupSelectQuery from '~/db/generateBTLookupSelectQuery'; import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2'; import conditionV2 from '~/db/conditionV2'; @@ -770,7 +769,7 @@ class BaseModelSqlv2 { } case UITypes.Lookup: { - const _selectQb = await generateBTLookupSelectQuery({ + const _selectQb = await generateMMLookupSelectQuery({ baseModelSqlv2: this, column, alias: null, diff --git a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts index 0af42ea1ef..aedb1e775a 100644 --- a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts @@ -10,7 +10,6 @@ import type { } from '~/models'; import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2'; -import { NcError } from '~/helpers/catchError'; import { getAliasGenerator } from '~/utils'; export default async function generateMMLookupSelectQuery({ @@ -24,7 +23,7 @@ export default async function generateMMLookupSelectQuery({ baseModelSqlv2: BaseModelSqlv2; alias: string; model: Model; - getAlias: ReturnType; + getAlias?: ReturnType; }): Promise { const knex = baseModelSqlv2.dbDriver; @@ -41,26 +40,107 @@ export default async function generateMMLookupSelectQuery({ await relationCol.getColOptions(); // if not belongs to then throw error as we don't support - if (relation.type !== RelationTypes.BELONGS_TO) { - // NcError.badRequest('HasMany/ManyToMany lookup is not supported'); + if (relation.type === RelationTypes.BELONGS_TO) { + const childColumn = await relation.getChildColumn(); + const parentColumn = await relation.getParentColumn(); + const childModel = await childColumn.getModel(); + await childModel.getColumns(); + const parentModel = await parentColumn.getModel(); + await parentModel.getColumns(); + + selectQb = knex( + `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`, + ).where( + `${alias}.${parentColumn.column_name}`, + knex.raw(`??`, [ + `${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ + childColumn.column_name + }`, + ]), + ); + } + + // if not belongs to then throw error as we don't support + else if (relation.type === RelationTypes.HAS_MANY) { + const childColumn = await relation.getChildColumn(); + const parentColumn = await relation.getParentColumn(); + const childModel = await childColumn.getModel(); + await childModel.getColumns(); + const parentModel = await parentColumn.getModel(); + await parentModel.getColumns(); + + selectQb = knex( + `${baseModelSqlv2.getTnPath(childModel.table_name)} as ${alias}`, + ).where( + `${alias}.${parentColumn.column_name}`, + knex.raw(`??`, [ + `${rootAlias || baseModelSqlv2.getTnPath(parentModel.table_name)}.${ + childColumn.column_name + }`, + ]), + ); + } + + // if not belongs to then throw error as we don't support + if (relation.type === RelationTypes.MANY_TO_MANY) { + const childColumn = await relation.getChildColumn(); + const parentColumn = await relation.getParentColumn(); + const childModel = await childColumn.getModel(); + await childModel.getColumns(); + const parentModel = await parentColumn.getModel(); + await parentModel.getColumns(); + + selectQb = knex( + `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`, + ).where( + `${alias}.${parentColumn.column_name}`, + knex.raw(`??`, [ + `${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ + childColumn.column_name + }`, + ]), + ); + + + + /* + const mmTableAlias = getAlias(); + + const mmModel = await relation.getMMModel(); + const mmChildCol = await relation.getMMChildColumn(); + const mmParentCol = await relation.getMMParentColumn(); + + // knex( + // `${baseModelSqlv2.getTnPath( + // parentModel?.table_name, + // )} as ${nestedAlias}`, + // ) + selectQb + .innerJoin( + baseModelSqlv2.getTnPath(mmModel.table_name), + knex.ref( + `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ + mmParentCol.column_name + }`, + ), + '=', + knex.ref(`${nestedAlias}.${parentColumn.column_name}`), + ) + .where( + knex.ref( + `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ + mmChildCol.column_name + }`, + ), + '=', + knex.ref( + `${alias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ + childColumn.column_name + }`, + ), + ); + * */ } - const childColumn = await relation.getChildColumn(); - const parentColumn = await relation.getParentColumn(); - const childModel = await childColumn.getModel(); - await childModel.getColumns(); - const parentModel = await parentColumn.getModel(); - await parentModel.getColumns(); - - selectQb = knex( - `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`, - ).where( - `${alias}.${parentColumn.column_name}`, - knex.raw(`??`, [ - `${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ - childColumn.column_name - }`, - ]), - ); } let lookupColumn = await lookup.getLookupColumn(); let prevAlias = alias; @@ -117,11 +197,12 @@ export default async function generateMMLookupSelectQuery({ const mmChildCol = await relation.getMMChildColumn(); const mmParentCol = await relation.getMMParentColumn(); - knex( - `${baseModelSqlv2.getTnPath( - parentModel?.table_name, - )} as ${nestedAlias}`, - ) + // knex( + // `${baseModelSqlv2.getTnPath( + // parentModel?.table_name, + // )} as ${nestedAlias}`, + // ) + selectQb .innerJoin( baseModelSqlv2.getTnPath(mmModel.table_name), knex.ref( @@ -213,12 +294,17 @@ export default async function generateMMLookupSelectQuery({ break; default: { - selectQb.select(`${prevAlias}.${lookupColumn.column_name}`); + selectQb.select(`${prevAlias}.${lookupColumn.column_name} as ${lookupColumn.title}`); } break; } - return { builder: selectQb }; + selectQb.orderBy(`${lookupColumn.title}`, 'asc'); + + // mysql : https://www.db-fiddle.com/f/qC4hrbSrSp7v3exb4moVCE/1 +const subQueryAlias = getAlias(); + // return { builder: knex.select(knex.raw('array_agg(??)', [lookupColumn.title])).from(selectQb) }; + return { builder: knex.select(knex.raw("cast(JSON_ARRAYAGG(??) as NCHAR)", [lookupColumn.title])).from(selectQb.as(subQueryAlias)) }; } } From e4e99e01e15501bc0b879019937188feb7a05da9 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:10 +0000 Subject: [PATCH 05/45] feat: group by - lookup support - mm/hm lookup - WIP --- .../src/db/generateMMLookupSelectQuery.ts | 109 ++++++++++-------- 1 file changed, 64 insertions(+), 45 deletions(-) diff --git a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts index aedb1e775a..8217f9585d 100644 --- a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts @@ -11,6 +11,7 @@ import type { import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2'; import { getAliasGenerator } from '~/utils'; +import { NcError } from '~/helpers/catchError'; export default async function generateMMLookupSelectQuery({ column, @@ -82,7 +83,7 @@ export default async function generateMMLookupSelectQuery({ } // if not belongs to then throw error as we don't support - if (relation.type === RelationTypes.MANY_TO_MANY) { + else if (relation.type === RelationTypes.MANY_TO_MANY) { const childColumn = await relation.getChildColumn(); const parentColumn = await relation.getParentColumn(); const childModel = await childColumn.getModel(); @@ -92,19 +93,9 @@ export default async function generateMMLookupSelectQuery({ selectQb = knex( `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`, - ).where( - `${alias}.${parentColumn.column_name}`, - knex.raw(`??`, [ - `${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ - childColumn.column_name - }`, - ]), ); - - - /* - const mmTableAlias = getAlias(); + const mmTableAlias = getAlias(); const mmModel = await relation.getMMModel(); const mmChildCol = await relation.getMMChildColumn(); @@ -117,29 +108,20 @@ export default async function generateMMLookupSelectQuery({ // ) selectQb .innerJoin( - baseModelSqlv2.getTnPath(mmModel.table_name), - knex.ref( - `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ - mmParentCol.column_name - }`, - ), + baseModelSqlv2.getTnPath(mmModel.table_name, mmTableAlias), + knex.ref(`${mmTableAlias}.${mmParentCol.column_name}`), '=', - knex.ref(`${nestedAlias}.${parentColumn.column_name}`), + knex.ref(`${alias}.${parentColumn.column_name}`), ) .where( - knex.ref( - `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ - mmChildCol.column_name - }`, - ), + knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`), '=', knex.ref( - `${alias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ - childColumn.column_name - }`, + `${ + rootAlias || baseModelSqlv2.getTnPath(childModel.table_name) + }.${childColumn.column_name}`, ), ); - * */ } } let lookupColumn = await lookup.getLookupColumn(); @@ -204,21 +186,13 @@ export default async function generateMMLookupSelectQuery({ // ) selectQb .innerJoin( - baseModelSqlv2.getTnPath(mmModel.table_name), - knex.ref( - `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ - mmParentCol.column_name - }`, - ), + baseModelSqlv2.getTnPath(mmModel.table_name, mmTableAlias), + knex.ref(`${mmTableAlias}.${mmParentCol.column_name}`), '=', knex.ref(`${nestedAlias}.${parentColumn.column_name}`), ) .where( - knex.ref( - `${baseModelSqlv2.getTnPath(mmModel.table_name)}.${ - mmChildCol.column_name - }`, - ), + knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`), '=', knex.ref( `${alias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ @@ -294,17 +268,62 @@ export default async function generateMMLookupSelectQuery({ break; default: { - selectQb.select(`${prevAlias}.${lookupColumn.column_name} as ${lookupColumn.title}`); + selectQb.select( + `${prevAlias}.${lookupColumn.column_name} as ${lookupColumn.title}`, + ); } break; } - selectQb.orderBy(`${lookupColumn.title}`, 'asc'); + const subQueryAlias = getAlias(); + + if (baseModelSqlv2.isPg) { + selectQb.orderBy(`${lookupColumn.title}`, 'asc'); + // // alternate approach with array_agg + // return { + // builder: knex + // .select(knex.raw('array_agg(??)', [lookupColumn.title])) + // .from(selectQb), + // }; + return { + builder: knex + .select( + knex.raw("STRING_AGG(DISTINCT ??::text, '___')", [ + lookupColumn.title, + ]), + ) + .from(selectQb.as(subQueryAlias)), + }; + } else if (baseModelSqlv2.isMySQL) { + // // alternate approach with JSON_ARRAYAGG + // return { + // builder: knex + // .select( + // knex.raw('cast(JSON_ARRAYAGG(??) as NCHAR)', [lookupColumn.title]), + // ) + // .from(selectQb.as(subQueryAlias)), + // }; + + return { + builder: knex + .select( + knex.raw( + "GROUP_CONCAT(DISTINCT ?? ORDER BY ?? ASC SEPARATOR '___')", + [lookupColumn.title, lookupColumn.title], + ), + ) + .from(selectQb.as(subQueryAlias)), + }; + } else if (baseModelSqlv2.isSqlite) { + selectQb.orderBy(`${lookupColumn.title}`, 'asc'); + return { + builder: knex + .select(knex.raw(`group_concat(??)`, [lookupColumn.title])) + .from(selectQb.as(subQueryAlias)), + }; + } - // mysql : https://www.db-fiddle.com/f/qC4hrbSrSp7v3exb4moVCE/1 -const subQueryAlias = getAlias(); - // return { builder: knex.select(knex.raw('array_agg(??)', [lookupColumn.title])).from(selectQb) }; - return { builder: knex.select(knex.raw("cast(JSON_ARRAYAGG(??) as NCHAR)", [lookupColumn.title])).from(selectQb.as(subQueryAlias)) }; + NcError.notImplemented('Database not supported Group by on Lookup'); } } From c968a15071478bbe850603e4f4372392fb7bb9ed Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:10 +0000 Subject: [PATCH 06/45] feat: group by - lookup support - mm/hm lookup - list filter - WIP --- packages/nc-gui/composables/useViewGroupBy.ts | 2 +- packages/nocodb/src/db/BaseModelSqlv2.ts | 5 +++++ packages/nocodb/src/db/conditionV2.ts | 22 +++++++++++++++++++ .../src/db/generateMMLookupSelectQuery.ts | 15 +++++++------ packages/nocodb/src/utils/globals.ts | 4 +++- 5 files changed, 39 insertions(+), 9 deletions(-) diff --git a/packages/nc-gui/composables/useViewGroupBy.ts b/packages/nc-gui/composables/useViewGroupBy.ts index 809578dd1c..7bbb31c3dc 100644 --- a/packages/nc-gui/composables/useViewGroupBy.ts +++ b/packages/nc-gui/composables/useViewGroupBy.ts @@ -150,7 +150,7 @@ export const useViewGroupBy = (view: Ref, where?: Computed } else if ([UITypes.Date, UITypes.DateTime].includes(curr.column_uidt as UITypes)) { acc += `${acc.length ? '~and' : ''}(${curr.title},eq,exactDate,${curr.key})` } else { - acc += `${acc.length ? '~and' : ''}(${curr.title},eq,${curr.key})` + acc += `${acc.length ? '~and' : ''}(${curr.title},gb_val,${curr.key})` } return acc }, existing) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 69a338a863..d2ecee27c8 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -386,6 +386,11 @@ class BaseModelSqlv2 { validateFormula: true, }); } + + console.log('=======data') + console.log(qb.toQuery()) + console.log('=======data') + return data?.map((d) => { d.__proto__ = proto; return d; diff --git a/packages/nocodb/src/db/conditionV2.ts b/packages/nocodb/src/db/conditionV2.ts index 4495be7cef..26de69773d 100644 --- a/packages/nocodb/src/db/conditionV2.ts +++ b/packages/nocodb/src/db/conditionV2.ts @@ -13,6 +13,9 @@ import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2'; import { sanitize } from '~/helpers/sqlSanitize'; import Filter from '~/models/Filter'; +import generateMMLookupSelectQuery from '~/db/generateMMLookupSelectQuery'; +import { Model } from '~/models'; +import { getAliasGenerator } from '~/utils'; // tod: tobe fixed // extend(customParseFormat); @@ -112,6 +115,25 @@ const parseConditionV2 = async ( }); }; } else { + // handle group by filter separately + if (filter.comparison_op === 'gb_val') { + const column = await filter.getColumn(); + const model = await column.getModel(); + const lkQb = await generateMMLookupSelectQuery({ + baseModelSqlv2, + alias: alias, + model, + column, + getAlias: getAliasGenerator('__gb_filter_lk'), + }); + return (qb) => { + qb.where( + knex.raw('?',[filter.value]), + lkQb.builder, + ); + }; + } + const column = await filter.getColumn(); if (!column) { if (throwErrorIfInvalid) { diff --git a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts index 8217f9585d..41b7ef4908 100644 --- a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts @@ -315,14 +315,15 @@ export default async function generateMMLookupSelectQuery({ ) .from(selectQb.as(subQueryAlias)), }; - } else if (baseModelSqlv2.isSqlite) { - selectQb.orderBy(`${lookupColumn.title}`, 'asc'); - return { - builder: knex - .select(knex.raw(`group_concat(??)`, [lookupColumn.title])) - .from(selectQb.as(subQueryAlias)), - }; } + // else if (baseModelSqlv2.isSqlite) { + // selectQb.orderBy(`${lookupColumn.title}`, 'asc'); + // return { + // builder: knex + // .select(knex.raw(`group_concat(??, '___')`, [lookupColumn.title])) + // .from(selectQb.as(subQueryAlias)), + // }; + // } NcError.notImplemented('Database not supported Group by on Lookup'); } diff --git a/packages/nocodb/src/utils/globals.ts b/packages/nocodb/src/utils/globals.ts index 786320dac7..2602637916 100644 --- a/packages/nocodb/src/utils/globals.ts +++ b/packages/nocodb/src/utils/globals.ts @@ -201,7 +201,9 @@ export const COMPARISON_OPS = [ 'isWithin', 'btw', 'nbtw', -]; + + 'gb_val' +] as any; export const IS_WITHIN_COMPARISON_SUB_OPS = [ 'pastWeek', From b6c0726948ba4341cea2ef5b623e916af91fa4b0 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:11 +0000 Subject: [PATCH 07/45] feat: sqlite support --- .../src/db/generateMMLookupSelectQuery.ts | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts index 41b7ef4908..db2959fdbe 100644 --- a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts @@ -315,15 +315,19 @@ export default async function generateMMLookupSelectQuery({ ) .from(selectQb.as(subQueryAlias)), }; + } else if (baseModelSqlv2.isSqlite) { + // ref: https://stackoverflow.com/questions/13382856/sqlite3-join-group-concat-using-distinct-with-custom-separator + selectQb.orderBy(`${lookupColumn.title}`, 'asc'); + return { + builder: knex + .select( + knex.raw(`replace(group_concat(distinct ??), ',', '___')`, [ + lookupColumn.title, + ]), + ) + .from(selectQb.as(subQueryAlias)), + }; } - // else if (baseModelSqlv2.isSqlite) { - // selectQb.orderBy(`${lookupColumn.title}`, 'asc'); - // return { - // builder: knex - // .select(knex.raw(`group_concat(??, '___')`, [lookupColumn.title])) - // .from(selectQb.as(subQueryAlias)), - // }; - // } NcError.notImplemented('Database not supported Group by on Lookup'); } From eece67be9729043bf2f742e1c70a67c6d83c45fd Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:11 +0000 Subject: [PATCH 08/45] feat: group by - avoid ignoring duplicate values from different records --- packages/nocodb/src/db/conditionV2.ts | 17 +++++++++-------- .../src/db/generateMMLookupSelectQuery.ts | 14 +++++--------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/nocodb/src/db/conditionV2.ts b/packages/nocodb/src/db/conditionV2.ts index 26de69773d..4ea2bc1dbd 100644 --- a/packages/nocodb/src/db/conditionV2.ts +++ b/packages/nocodb/src/db/conditionV2.ts @@ -118,20 +118,21 @@ const parseConditionV2 = async ( // handle group by filter separately if (filter.comparison_op === 'gb_val') { const column = await filter.getColumn(); - const model = await column.getModel(); - const lkQb = await generateMMLookupSelectQuery({ + if (column.uidt === UITypes.Lookup) { + const model = await column.getModel(); + const lkQb = await generateMMLookupSelectQuery({ baseModelSqlv2, alias: alias, model, column, getAlias: getAliasGenerator('__gb_filter_lk'), }); - return (qb) => { - qb.where( - knex.raw('?',[filter.value]), - lkQb.builder, - ); - }; + return (qb) => { + qb.where(knex.raw('?', [filter.value]), lkQb.builder); + }; + } else { + filter.comparison_op = 'eq'; + } } const column = await filter.getColumn(); diff --git a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts index db2959fdbe..64f33c17d5 100644 --- a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateMMLookupSelectQuery.ts @@ -308,10 +308,10 @@ export default async function generateMMLookupSelectQuery({ return { builder: knex .select( - knex.raw( - "GROUP_CONCAT(DISTINCT ?? ORDER BY ?? ASC SEPARATOR '___')", - [lookupColumn.title, lookupColumn.title], - ), + knex.raw("GROUP_CONCAT(?? ORDER BY ?? ASC SEPARATOR '___')", [ + lookupColumn.title, + lookupColumn.title, + ]), ) .from(selectQb.as(subQueryAlias)), }; @@ -320,11 +320,7 @@ export default async function generateMMLookupSelectQuery({ selectQb.orderBy(`${lookupColumn.title}`, 'asc'); return { builder: knex - .select( - knex.raw(`replace(group_concat(distinct ??), ',', '___')`, [ - lookupColumn.title, - ]), - ) + .select(knex.raw(`group_concat(??, '___')`, [lookupColumn.title])) .from(selectQb.as(subQueryAlias)), }; } From 23c80f7c417b6eed13d2934aa6acc6453401dbe4 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:11 +0000 Subject: [PATCH 09/45] refactor: rename function name --- packages/nocodb/src/db/BaseModelSqlv2.ts | 6 +- packages/nocodb/src/db/conditionV2.ts | 4 +- .../src/db/generateBTLookupSelectQuery.ts | 164 ------------------ ...tQuery.ts => generateLookupSelectQuery.ts} | 4 +- 4 files changed, 7 insertions(+), 171 deletions(-) delete mode 100644 packages/nocodb/src/db/generateBTLookupSelectQuery.ts rename packages/nocodb/src/db/{generateMMLookupSelectQuery.ts => generateLookupSelectQuery.ts} (98%) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index d2ecee27c8..dadbe3b690 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -63,7 +63,7 @@ import { } from '~/utils/globals'; import { extractProps } from '~/helpers/extractProps'; import { defaultLimitConfig } from '~/helpers/extractLimitAndOffset'; -import generateMMLookupSelectQuery from "~/db/generateMMLookupSelectQuery"; +import generateLookupSelectQuery from "~/db/generateLookupSelectQuery"; dayjs.extend(utc); @@ -605,7 +605,7 @@ class BaseModelSqlv2 { break; case UITypes.Lookup: { - const _selectQb = await generateMMLookupSelectQuery({ + const _selectQb = await generateLookupSelectQuery({ baseModelSqlv2: this, column, alias: null, @@ -774,7 +774,7 @@ class BaseModelSqlv2 { } case UITypes.Lookup: { - const _selectQb = await generateMMLookupSelectQuery({ + const _selectQb = await generateLookupSelectQuery({ baseModelSqlv2: this, column, alias: null, diff --git a/packages/nocodb/src/db/conditionV2.ts b/packages/nocodb/src/db/conditionV2.ts index 4ea2bc1dbd..b352535226 100644 --- a/packages/nocodb/src/db/conditionV2.ts +++ b/packages/nocodb/src/db/conditionV2.ts @@ -13,7 +13,7 @@ import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2'; import { sanitize } from '~/helpers/sqlSanitize'; import Filter from '~/models/Filter'; -import generateMMLookupSelectQuery from '~/db/generateMMLookupSelectQuery'; +import generateLookupSelectQuery from '~/db/generateLookupSelectQuery'; import { Model } from '~/models'; import { getAliasGenerator } from '~/utils'; @@ -120,7 +120,7 @@ const parseConditionV2 = async ( const column = await filter.getColumn(); if (column.uidt === UITypes.Lookup) { const model = await column.getModel(); - const lkQb = await generateMMLookupSelectQuery({ + const lkQb = await generateLookupSelectQuery({ baseModelSqlv2, alias: alias, model, diff --git a/packages/nocodb/src/db/generateBTLookupSelectQuery.ts b/packages/nocodb/src/db/generateBTLookupSelectQuery.ts deleted file mode 100644 index 5883fd0b56..0000000000 --- a/packages/nocodb/src/db/generateBTLookupSelectQuery.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { RelationTypes, UITypes } from 'nocodb-sdk'; -import type LookupColumn from '../models/LookupColumn'; -import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; -import type { - Column, - FormulaColumn, - LinkToAnotherRecordColumn, - Model, - RollupColumn, -} from '~/models'; -import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; -import genRollupSelectv2 from '~/db/genRollupSelectv2'; -import { NcError } from '~/helpers/catchError'; - -export default async function generateBTLookupSelectQuery({ - column, - baseModelSqlv2, - alias, - model, -}: { - column: Column; - baseModelSqlv2: BaseModelSqlv2; - alias: string; - model: Model; -}): Promise { - const knex = baseModelSqlv2.dbDriver; - - const rootAlias = alias; - - { - let aliasCount = 0, - selectQb; - const alias = `__nc_lk_${aliasCount++}`; - const lookup = await column.getColOptions(); - { - const relationCol = await lookup.getRelationColumn(); - const relation = - await relationCol.getColOptions(); - - // if not belongs to then throw error as we don't support - if (relation.type !== RelationTypes.BELONGS_TO) { - // NcError.badRequest('HasMany/ManyToMany lookup is not supported'); - } - const childColumn = await relation.getChildColumn(); - const parentColumn = await relation.getParentColumn(); - const childModel = await childColumn.getModel(); - await childModel.getColumns(); - const parentModel = await parentColumn.getModel(); - await parentModel.getColumns(); - - selectQb = knex( - `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`, - ).where( - `${alias}.${parentColumn.column_name}`, - knex.raw(`??`, [ - `${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${ - childColumn.column_name - }`, - ]), - ); - } - let lookupColumn = await lookup.getLookupColumn(); - let prevAlias = alias; - while (lookupColumn.uidt === UITypes.Lookup) { - const nestedAlias = `__nc_lk_nested_${aliasCount++}`; - const nestedLookup = await lookupColumn.getColOptions(); - const relationCol = await nestedLookup.getRelationColumn(); - const relation = - await relationCol.getColOptions(); - - // if any of the relation in nested lookup is - // not belongs to then throw error as we don't support - if (relation.type !== RelationTypes.BELONGS_TO) { - // NcError.badRequest('HasMany/ManyToMany lookup is not supported'); - } - - const childColumn = await relation.getChildColumn(); - const parentColumn = await relation.getParentColumn(); - const childModel = await childColumn.getModel(); - await childModel.getColumns(); - const parentModel = await parentColumn.getModel(); - await parentModel.getColumns(); - - selectQb.join( - `${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${nestedAlias}`, - `${nestedAlias}.${parentColumn.column_name}`, - `${prevAlias}.${childColumn.column_name}`, - ); - - lookupColumn = await nestedLookup.getLookupColumn(); - prevAlias = nestedAlias; - } - - switch (lookupColumn.uidt) { - case UITypes.Links: - case UITypes.Rollup: - { - const builder = ( - await genRollupSelectv2({ - baseModelSqlv2, - knex, - columnOptions: - (await lookupColumn.getColOptions()) as RollupColumn, - alias: prevAlias, - }) - ).builder; - selectQb.select(builder); - } - break; - case UITypes.LinkToAnotherRecord: - { - const nestedAlias = `__nc_sort${aliasCount++}`; - const relation = - await lookupColumn.getColOptions(); - if (relation.type !== 'bt') return; - - const colOptions = - (await column.getColOptions()) as LinkToAnotherRecordColumn; - const childColumn = await colOptions.getChildColumn(); - const parentColumn = await colOptions.getParentColumn(); - const childModel = await childColumn.getModel(); - await childModel.getColumns(); - const parentModel = await parentColumn.getModel(); - await parentModel.getColumns(); - - selectQb - .join( - `${baseModelSqlv2.getTnPath( - parentModel.table_name, - )} as ${nestedAlias}`, - `${nestedAlias}.${parentColumn.column_name}`, - `${prevAlias}.${childColumn.column_name}`, - ) - .select(parentModel?.displayValue?.column_name); - } - break; - case UITypes.Formula: - { - const builder = ( - await formulaQueryBuilderv2( - baseModelSqlv2, - ( - await column.getColOptions() - ).formula, - null, - model, - column, - ) - ).builder; - - selectQb.select(builder); - } - break; - default: - { - selectQb.select(`${prevAlias}.${lookupColumn.column_name}`); - } - - break; - } - - return { builder: selectQb }; - } -} diff --git a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts b/packages/nocodb/src/db/generateLookupSelectQuery.ts similarity index 98% rename from packages/nocodb/src/db/generateMMLookupSelectQuery.ts rename to packages/nocodb/src/db/generateLookupSelectQuery.ts index 64f33c17d5..f55afe0f12 100644 --- a/packages/nocodb/src/db/generateMMLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateLookupSelectQuery.ts @@ -13,7 +13,7 @@ import genRollupSelectv2 from '~/db/genRollupSelectv2'; import { getAliasGenerator } from '~/utils'; import { NcError } from '~/helpers/catchError'; -export default async function generateMMLookupSelectQuery({ +export default async function generateLookupSelectQuery({ column, baseModelSqlv2, alias, @@ -289,7 +289,7 @@ export default async function generateMMLookupSelectQuery({ return { builder: knex .select( - knex.raw("STRING_AGG(DISTINCT ??::text, '___')", [ + knex.raw("STRING_AGG(??::text, '___')", [ lookupColumn.title, ]), ) From 532d603a2df1ca9a36b2cf7ec3f4482c7f106cd2 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:11 +0000 Subject: [PATCH 10/45] refactor: split value and display --- .../nc-gui/components/smartsheet/grid/GroupBy.vue | 2 +- .../nocodb/src/db/generateLookupSelectQuery.ts | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/grid/GroupBy.vue b/packages/nc-gui/components/smartsheet/grid/GroupBy.vue index 67919f0934..dbb97ad07a 100644 --- a/packages/nc-gui/components/smartsheet/grid/GroupBy.vue +++ b/packages/nc-gui/components/smartsheet/grid/GroupBy.vue @@ -247,7 +247,7 @@ const onScroll = (e: Event) => { 'font-weight': 500, }" > - {{ grp.key in GROUP_BY_VARS.VAR_TITLES ? GROUP_BY_VARS.VAR_TITLES[grp.key] : grp.key }} + {{ grp.key in GROUP_BY_VARS.VAR_TITLES ? GROUP_BY_VARS.VAR_TITLES[grp.key] : grp.key.replaceAll('___',', ') }} diff --git a/packages/nocodb/src/db/generateLookupSelectQuery.ts b/packages/nocodb/src/db/generateLookupSelectQuery.ts index f55afe0f12..3529375fba 100644 --- a/packages/nocodb/src/db/generateLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateLookupSelectQuery.ts @@ -13,6 +13,8 @@ import genRollupSelectv2 from '~/db/genRollupSelectv2'; import { getAliasGenerator } from '~/utils'; import { NcError } from '~/helpers/catchError'; +const LOOKUP_VAL_SEPARATOR = '___'; + export default async function generateLookupSelectQuery({ column, baseModelSqlv2, @@ -289,8 +291,9 @@ export default async function generateLookupSelectQuery({ return { builder: knex .select( - knex.raw("STRING_AGG(??::text, '___')", [ + knex.raw('STRING_AGG(??::text, ?)', [ lookupColumn.title, + LOOKUP_VAL_SEPARATOR, ]), ) .from(selectQb.as(subQueryAlias)), @@ -308,9 +311,10 @@ export default async function generateLookupSelectQuery({ return { builder: knex .select( - knex.raw("GROUP_CONCAT(?? ORDER BY ?? ASC SEPARATOR '___')", [ + knex.raw('GROUP_CONCAT(?? ORDER BY ?? ASC SEPARATOR ?)', [ lookupColumn.title, lookupColumn.title, + LOOKUP_VAL_SEPARATOR, ]), ) .from(selectQb.as(subQueryAlias)), @@ -320,7 +324,12 @@ export default async function generateLookupSelectQuery({ selectQb.orderBy(`${lookupColumn.title}`, 'asc'); return { builder: knex - .select(knex.raw(`group_concat(??, '___')`, [lookupColumn.title])) + .select( + knex.raw(`group_concat(??, ?)`, [ + lookupColumn.title, + LOOKUP_VAL_SEPARATOR, + ]), + ) .from(selectQb.as(subQueryAlias)), }; } From e1995c8f54f768cbe9771178961d65ebc9293d05 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:11 +0000 Subject: [PATCH 11/45] fix: join query corrections --- .../nocodb/src/db/generateLookupSelectQuery.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/nocodb/src/db/generateLookupSelectQuery.ts b/packages/nocodb/src/db/generateLookupSelectQuery.ts index 3529375fba..ccfef20e35 100644 --- a/packages/nocodb/src/db/generateLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateLookupSelectQuery.ts @@ -75,10 +75,10 @@ export default async function generateLookupSelectQuery({ selectQb = knex( `${baseModelSqlv2.getTnPath(childModel.table_name)} as ${alias}`, ).where( - `${alias}.${parentColumn.column_name}`, + `${alias}.${childColumn.column_name}`, knex.raw(`??`, [ `${rootAlias || baseModelSqlv2.getTnPath(parentModel.table_name)}.${ - childColumn.column_name + parentColumn.column_name }`, ]), ); @@ -103,11 +103,6 @@ export default async function generateLookupSelectQuery({ const mmChildCol = await relation.getMMChildColumn(); const mmParentCol = await relation.getMMParentColumn(); - // knex( - // `${baseModelSqlv2.getTnPath( - // parentModel?.table_name, - // )} as ${nestedAlias}`, - // ) selectQb .innerJoin( baseModelSqlv2.getTnPath(mmModel.table_name, mmTableAlias), @@ -162,10 +157,10 @@ export default async function generateLookupSelectQuery({ selectQb.join( `${baseModelSqlv2.getTnPath( - childModel.table_name, + parentModel.table_name, )} as ${nestedAlias}`, `${nestedAlias}.${parentColumn.column_name}`, - `${prevAlias}.${childColumn.column_name}`, + `${prevAlias}.${parentColumn.column_name}`, ); } else if (relation.type === RelationTypes.MANY_TO_MANY) { const childColumn = await relation.getChildColumn(); @@ -226,7 +221,7 @@ export default async function generateLookupSelectQuery({ break; case UITypes.LinkToAnotherRecord: { - const nestedAlias = `__nc_sort${aliasCount++}`; + const nestedAlias = getAlias(); const relation = await lookupColumn.getColOptions(); if (relation.type !== 'bt') return; From b0a38676cc5b1048e27c3871e08544164f3f284f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:11 +0000 Subject: [PATCH 12/45] fix: ignore row ordering and fix nested mm lookup bug --- packages/nocodb/src/db/generateLookupSelectQuery.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/nocodb/src/db/generateLookupSelectQuery.ts b/packages/nocodb/src/db/generateLookupSelectQuery.ts index ccfef20e35..8b0cad3b70 100644 --- a/packages/nocodb/src/db/generateLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateLookupSelectQuery.ts @@ -184,10 +184,15 @@ export default async function generateLookupSelectQuery({ selectQb .innerJoin( baseModelSqlv2.getTnPath(mmModel.table_name, mmTableAlias), - knex.ref(`${mmTableAlias}.${mmParentCol.column_name}`), + knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`), '=', - knex.ref(`${nestedAlias}.${parentColumn.column_name}`), + knex.ref(`${prevAlias}.${childColumn.column_name}`), ) + .innerJoin(knex.raw('?? as ??', [baseModelSqlv2.getTnPath(parentModel.table_name), nestedAlias]), + knex.ref(`${mmTableAlias}.${mmParentCol.column_name}`), + '=', + knex.ref(`${nestedAlias}.${parentColumn.column_name}`), + ) .where( knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`), '=', @@ -276,7 +281,7 @@ export default async function generateLookupSelectQuery({ const subQueryAlias = getAlias(); if (baseModelSqlv2.isPg) { - selectQb.orderBy(`${lookupColumn.title}`, 'asc'); + // selectQb.orderBy(`${lookupColumn.title}`, 'asc'); // // alternate approach with array_agg // return { // builder: knex @@ -316,7 +321,7 @@ export default async function generateLookupSelectQuery({ }; } else if (baseModelSqlv2.isSqlite) { // ref: https://stackoverflow.com/questions/13382856/sqlite3-join-group-concat-using-distinct-with-custom-separator - selectQb.orderBy(`${lookupColumn.title}`, 'asc'); + // selectQb.orderBy(`${lookupColumn.title}`, 'asc'); return { builder: knex .select( From e491326a3abbbbcc12c91ceefb970f7d7ce91831 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:11 +0000 Subject: [PATCH 13/45] fix: fix nested hm lookup bug --- packages/nocodb/src/db/generateLookupSelectQuery.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/db/generateLookupSelectQuery.ts b/packages/nocodb/src/db/generateLookupSelectQuery.ts index 8b0cad3b70..d85c3cc430 100644 --- a/packages/nocodb/src/db/generateLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateLookupSelectQuery.ts @@ -157,9 +157,9 @@ export default async function generateLookupSelectQuery({ selectQb.join( `${baseModelSqlv2.getTnPath( - parentModel.table_name, + childModel.table_name, )} as ${nestedAlias}`, - `${nestedAlias}.${parentColumn.column_name}`, + `${nestedAlias}.${childColumn.column_name}`, `${prevAlias}.${parentColumn.column_name}`, ); } else if (relation.type === RelationTypes.MANY_TO_MANY) { From c382d66494443d5d8c785f10e6fb5100ea938932 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:12 +0000 Subject: [PATCH 14/45] fix: support Lookup to LTAR --- .../src/db/generateLookupSelectQuery.ts | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/packages/nocodb/src/db/generateLookupSelectQuery.ts b/packages/nocodb/src/db/generateLookupSelectQuery.ts index d85c3cc430..681e08a899 100644 --- a/packages/nocodb/src/db/generateLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateLookupSelectQuery.ts @@ -8,6 +8,7 @@ import type { Model, RollupColumn, } from '~/models'; +import type { LinksColumn } from '~/models'; import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2'; import { getAliasGenerator } from '~/utils'; @@ -33,8 +34,7 @@ export default async function generateLookupSelectQuery({ const rootAlias = alias; { - let aliasCount = 0, - selectQb; + let selectQb; const alias = getAlias(); const lookup = await column.getColOptions(); { @@ -123,10 +123,22 @@ export default async function generateLookupSelectQuery({ } let lookupColumn = await lookup.getLookupColumn(); let prevAlias = alias; - while (lookupColumn.uidt === UITypes.Lookup) { + while ( + lookupColumn.uidt === UITypes.Lookup || + lookupColumn.uidt === UITypes.LinkToAnotherRecord + ) { const nestedAlias = getAlias(); - const nestedLookup = await lookupColumn.getColOptions(); - const relationCol = await nestedLookup.getRelationColumn(); + + let relationCol: Column; + let nestedLookupColOpt: LookupColumn; + + if (lookupColumn.uidt === UITypes.Lookup) { + nestedLookupColOpt = await lookupColumn.getColOptions(); + relationCol = await nestedLookupColOpt.getRelationColumn(); + } else { + relationCol = lookupColumn; + } + const relation = await relationCol.getColOptions(); @@ -188,11 +200,15 @@ export default async function generateLookupSelectQuery({ '=', knex.ref(`${prevAlias}.${childColumn.column_name}`), ) - .innerJoin(knex.raw('?? as ??', [baseModelSqlv2.getTnPath(parentModel.table_name), nestedAlias]), - knex.ref(`${mmTableAlias}.${mmParentCol.column_name}`), - '=', - knex.ref(`${nestedAlias}.${parentColumn.column_name}`), - ) + .innerJoin( + knex.raw('?? as ??', [ + baseModelSqlv2.getTnPath(parentModel.table_name), + nestedAlias, + ]), + knex.ref(`${mmTableAlias}.${mmParentCol.column_name}`), + '=', + knex.ref(`${nestedAlias}.${parentColumn.column_name}`), + ) .where( knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`), '=', @@ -204,7 +220,14 @@ export default async function generateLookupSelectQuery({ ); } - lookupColumn = await nestedLookup.getLookupColumn(); + if (lookupColumn.uidt === UITypes.Lookup) + lookupColumn = await nestedLookupColOpt.getLookupColumn(); + else + lookupColumn = await relationCol + .getColOptions() + .then((colOpt) => colOpt.getRelatedTable()) + .then((model) => model.getColumns()) + .then((cols) => cols.find((col) => col.pv)); prevAlias = nestedAlias; } @@ -223,7 +246,7 @@ export default async function generateLookupSelectQuery({ ).builder; selectQb.select(builder); } - break; + break; /* case UITypes.LinkToAnotherRecord: { const nestedAlias = getAlias(); @@ -250,7 +273,7 @@ export default async function generateLookupSelectQuery({ ) .select(parentModel?.displayValue?.column_name); } - break; + break;*/ case UITypes.Formula: { const builder = ( From 2b4eb320ed686b0a3135a7de9d8858dbf1cbf393 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:12 +0000 Subject: [PATCH 15/45] fix: get aggregated value as json string array to avoid issues with separator --- .../src/db/generateLookupSelectQuery.ts | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/packages/nocodb/src/db/generateLookupSelectQuery.ts b/packages/nocodb/src/db/generateLookupSelectQuery.ts index 681e08a899..0cfb993d82 100644 --- a/packages/nocodb/src/db/generateLookupSelectQuery.ts +++ b/packages/nocodb/src/db/generateLookupSelectQuery.ts @@ -304,44 +304,52 @@ export default async function generateLookupSelectQuery({ const subQueryAlias = getAlias(); if (baseModelSqlv2.isPg) { - // selectQb.orderBy(`${lookupColumn.title}`, 'asc'); - // // alternate approach with array_agg - // return { - // builder: knex - // .select(knex.raw('array_agg(??)', [lookupColumn.title])) - // .from(selectQb), - // }; + selectQb.orderBy(`${lookupColumn.title}`, 'asc'); + // alternate approach with array_agg return { builder: knex - .select( - knex.raw('STRING_AGG(??::text, ?)', [ - lookupColumn.title, - LOOKUP_VAL_SEPARATOR, - ]), - ) - .from(selectQb.as(subQueryAlias)), + .select(knex.raw('json_agg(??)::text', [lookupColumn.title])) + .from(selectQb), }; - } else if (baseModelSqlv2.isMySQL) { - // // alternate approach with JSON_ARRAYAGG + /* + // alternate approach with array_agg + return { + builder: knex + .select(knex.raw('array_agg(??)', [lookupColumn.title])) + .from(selectQb), + };*/ + // alternate approach with string aggregation // return { // builder: knex // .select( - // knex.raw('cast(JSON_ARRAYAGG(??) as NCHAR)', [lookupColumn.title]), + // knex.raw('STRING_AGG(??::text, ?)', [ + // lookupColumn.title, + // LOOKUP_VAL_SEPARATOR, + // ]), // ) // .from(selectQb.as(subQueryAlias)), // }; - + } else if (baseModelSqlv2.isMySQL) { + // alternate approach with JSON_ARRAYAGG return { builder: knex .select( - knex.raw('GROUP_CONCAT(?? ORDER BY ?? ASC SEPARATOR ?)', [ - lookupColumn.title, - lookupColumn.title, - LOOKUP_VAL_SEPARATOR, - ]), + knex.raw('cast(JSON_ARRAYAGG(??) as NCHAR)', [lookupColumn.title]), ) .from(selectQb.as(subQueryAlias)), }; + + // return { + // builder: knex + // .select( + // knex.raw('GROUP_CONCAT(?? ORDER BY ?? ASC SEPARATOR ?)', [ + // lookupColumn.title, + // lookupColumn.title, + // LOOKUP_VAL_SEPARATOR, + // ]), + // ) + // .from(selectQb.as(subQueryAlias)), + // }; } else if (baseModelSqlv2.isSqlite) { // ref: https://stackoverflow.com/questions/13382856/sqlite3-join-group-concat-using-distinct-with-custom-separator // selectQb.orderBy(`${lookupColumn.title}`, 'asc'); From 15fe013de274250faaaffdc11967e89e673ff1e6 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 16 Nov 2023 12:24:12 +0000 Subject: [PATCH 16/45] feat: handle json array parse in ui --- .../components/smartsheet/grid/GroupBy.vue | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/nc-gui/components/smartsheet/grid/GroupBy.vue b/packages/nc-gui/components/smartsheet/grid/GroupBy.vue index dbb97ad07a..89eb9e9159 100644 --- a/packages/nc-gui/components/smartsheet/grid/GroupBy.vue +++ b/packages/nc-gui/components/smartsheet/grid/GroupBy.vue @@ -1,5 +1,6 @@