Browse Source

Merge pull request #6987 from nocodb/nc-feat/lookup-group-by-hm-mm

Nc feat/lookup group by hm mm
pull/7001/head
Pranav C 12 months ago committed by GitHub
parent
commit
10e07377fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 39
      packages/nc-gui/components/smartsheet/grid/GroupBy.vue
  2. 28
      packages/nc-gui/components/smartsheet/grid/GroupByLabel.vue
  3. 48
      packages/nc-gui/components/smartsheet/toolbar/GroupByMenu.vue
  4. 10
      packages/nc-gui/composables/useViewGroupBy.ts
  5. 56
      packages/nocodb/src/db/BaseModelSqlv2.ts
  6. 40
      packages/nocodb/src/db/conditionV2.ts
  7. 163
      packages/nocodb/src/db/generateBTLookupSelectQuery.ts
  8. 399
      packages/nocodb/src/db/generateLookupSelectQuery.ts
  9. 5
      packages/nocodb/src/utils/globals.ts
  10. 10
      packages/nocodb/tests/unit/rest/tests/groupby.test.ts

39
packages/nc-gui/components/smartsheet/grid/GroupBy.vue

@ -1,8 +1,10 @@
<script lang="ts" setup> <script lang="ts" setup>
import tinycolor from 'tinycolor2' import tinycolor from 'tinycolor2'
import { UITypes } from 'nocodb-sdk'
import Table from './Table.vue' import Table from './Table.vue'
import GroupBy from './GroupBy.vue' import GroupBy from './GroupBy.vue'
import GroupByTable from './GroupByTable.vue' import GroupByTable from './GroupByTable.vue'
import GroupByLabel from './GroupByLabel.vue'
import { GROUP_BY_VARS, computed, ref } from '#imports' import { GROUP_BY_VARS, computed, ref } from '#imports'
import type { Group, Row } from '#imports' import type { Group, Row } from '#imports'
@ -134,6 +136,27 @@ const onScroll = (e: Event) => {
if (!vGroup.value.root) return if (!vGroup.value.root) return
_scrollLeft.value = (e.target as HTMLElement).scrollLeft _scrollLeft.value = (e.target as HTMLElement).scrollLeft
} }
// a method to parse group key if grouped column type is LTAR or Lookup
// in these 2 scenario it will return json array or `___` separated value
const parseKey = (group) => {
const key = group.key.toString()
// parse json array key if it's a lookup or link to another record
if ((key && group.column?.uidt === UITypes.Lookup) || group.column?.uidt === UITypes.LinkToAnotherRecord) {
try {
const parsedKey = JSON.parse(key)
return parsedKey
} catch {
// if parsing try to split it by `___` (for sqlite)
return key.split('___')
}
}
return [key]
}
const shouldRenderCell = (column) =>
[UITypes.Lookup, UITypes.Attachment, UITypes.Barcode, UITypes.QrCode, UITypes.Links].includes(column?.uidt)
</script> </script>
<template> <template>
@ -227,6 +250,15 @@ const onScroll = (e: Event) => {
</span> </span>
</a-tag> </a-tag>
</template> </template>
<div
v-else-if="!(grp.key in GROUP_BY_VARS.VAR_TITLES) && shouldRenderCell(grp.column)"
class="flex min-w-[100px] flex-wrap"
>
<template v-for="(val, ind) of parseKey(grp)" :key="ind">
<GroupByLabel v-if="val" :column="grp.column" :model-value="val" />
<span v-else class="text-gray-400">No mapped value</span>
</template>
</div>
<a-tag <a-tag
v-else v-else
:key="`panel-tag-${grp.column.id}-${grp.key}`" :key="`panel-tag-${grp.column.id}-${grp.key}`"
@ -247,7 +279,12 @@ const onScroll = (e: Event) => {
'font-weight': 500, 'font-weight': 500,
}" }"
> >
{{ grp.key in GROUP_BY_VARS.VAR_TITLES ? GROUP_BY_VARS.VAR_TITLES[grp.key] : grp.key }} <template v-if="grp.key in GROUP_BY_VARS.VAR_TITLES">{{
GROUP_BY_VARS.VAR_TITLES[grp.key]
}}</template>
<template v-else>
{{ parseKey(grp)?.join(', ') }}
</template>
</span> </span>
</a-tag> </a-tag>
</div> </div>

28
packages/nc-gui/components/smartsheet/grid/GroupByLabel.vue

@ -0,0 +1,28 @@
<script setup lang="ts">
import type { ColumnType } from 'nocodb-sdk'
import { isVirtualCol } from 'nocodb-sdk'
defineProps<{
column: ColumnType
modelValue: any
}>()
provide(ReadonlyInj, true)
</script>
<template>
<div class="pointer-events-none">
<LazySmartsheetRow :row="{ row: { [column.title]: modelValue }, rowMeta: {} }">
<LazySmartsheetVirtualCell v-if="isVirtualCol(column)" :model-value="modelValue" class="!text-gray-600" :column="column" />
<LazySmartsheetCell
v-else
:model-value="modelValue"
class="!text-gray-600"
:column="column"
:edit-enabled="false"
:read-only="true"
/>
</LazySmartsheetRow>
</div>
</template>

48
packages/nc-gui/components/smartsheet/toolbar/GroupByMenu.vue

@ -15,20 +15,10 @@ import {
useNuxtApp, useNuxtApp,
useSmartsheetStoreOrThrow, useSmartsheetStoreOrThrow,
useViewColumnsOrThrow, useViewColumnsOrThrow,
watch,
} from '#imports' } from '#imports'
const groupingUidt = [ const excludedGroupingUidt = [UITypes.Attachment]
UITypes.SingleSelect,
UITypes.MultiSelect,
UITypes.Checkbox,
UITypes.Date,
UITypes.SingleLineText,
UITypes.Number,
UITypes.Rollup,
UITypes.Lookup,
UITypes.Links,
UITypes.Formula,
]
const meta = inject(MetaInj, ref()) const meta = inject(MetaInj, ref())
const view = inject(ActiveViewInj, ref()) const view = inject(ActiveViewInj, ref())
@ -62,16 +52,16 @@ const groupedByColumnIds = computed(() => groupBy.value.map((g) => g.fk_column_i
const { eventBus } = useSmartsheetStoreOrThrow() const { eventBus } = useSmartsheetStoreOrThrow()
const { isMobileMode } = useGlobal() const { isMobileMode } = useGlobal()
const btLookups = ref([]) const supportedLookups = ref([])
const fieldsToGroupBy = computed(() => { const fieldsToGroupBy = computed(() => {
const fields = meta.value?.columns || [] const fields = meta.value?.columns || []
return fields.filter((field) => { return fields.filter((field) => {
if (!groupingUidt.includes(field.uidt as UITypes)) return false if (excludedGroupingUidt.includes(field.uidt as UITypes)) return false
if (field.uidt === UITypes.Lookup) { if (field.uidt === UITypes.Lookup) {
return btLookups.value.includes(field.id) return supportedLookups.value.includes(field.id)
} }
return true return true
@ -161,25 +151,18 @@ watch(open, () => {
} }
}) })
const loadBtLookups = async () => { const loadAllowedLookups = async () => {
const filteredLookupCols = [] const filteredLookupCols = []
try { try {
for (const col of meta.value?.columns || []) { for (const col of meta.value?.columns || []) {
if (col.uidt !== UITypes.Lookup) continue if (col.uidt !== UITypes.Lookup) continue
let nextCol = col let nextCol = col
let btLookup = true // check the lookup column is supported type or not
while (nextCol && nextCol.uidt === UITypes.Lookup) {
// check all the relation of nested lookup columns is bt or not
// include the column only if all only if all relations are bt
while (btLookup && nextCol && nextCol.uidt === UITypes.Lookup) {
const lookupRelation = (await getMeta(nextCol.fk_model_id))?.columns?.find( const lookupRelation = (await getMeta(nextCol.fk_model_id))?.columns?.find(
(c) => c.id === (nextCol.colOptions as LookupType).fk_relation_column_id, (c) => c.id === (nextCol.colOptions as LookupType).fk_relation_column_id,
) )
if ((lookupRelation.colOptions as LinkToAnotherRecordType).type !== RelationTypes.BELONGS_TO) {
btLookup = false
continue
}
const relatedTableMeta = await getMeta((lookupRelation.colOptions as LinkToAnotherRecordType).fk_related_model_id) const relatedTableMeta = await getMeta((lookupRelation.colOptions as LinkToAnotherRecordType).fk_related_model_id)
@ -190,22 +173,25 @@ const loadBtLookups = async () => {
// if next column is same as root lookup column then break the loop // if next column is same as root lookup column then break the loop
// since it's going to be a circular loop, and ignore the column // since it's going to be a circular loop, and ignore the column
if (nextCol.id === col.id) { if (nextCol.id === col.id) {
btLookup = false
break break
} }
} }
if (btLookup) filteredLookupCols.push(col.id) if (nextCol.uidt !== UITypes.Attachment) filteredLookupCols.push(col.id)
} }
btLookups.value = filteredLookupCols supportedLookups.value = filteredLookupCols
} catch (e) { } catch (e) {
console.error(e) console.error(e)
} }
} }
onMounted(async () => { onMounted(async () => {
await loadBtLookups() await loadAllowedLookups()
})
watch(meta, async () => {
await loadAllowedLookups()
}) })
</script> </script>
@ -242,9 +228,7 @@ onMounted(async () => {
<LazySmartsheetToolbarFieldListAutoCompleteDropdown <LazySmartsheetToolbarFieldListAutoCompleteDropdown
v-model="group.fk_column_id" v-model="group.fk_column_id"
class="caption nc-sort-field-select" class="caption nc-sort-field-select"
:columns=" :columns="fieldsToGroupBy"
fieldsToGroupBy.filter((f) => (f.id && !groupedByColumnIds.includes(f.id)) || f.id === group.fk_column_id)
"
:allow-empty="true" :allow-empty="true"
@change="saveGroupBy" @change="saveGroupBy"
@click.stop @click.stop

10
packages/nc-gui/composables/useViewGroupBy.ts

@ -89,6 +89,12 @@ export const useViewGroupBy = (view: Ref<ViewType | undefined>, where?: Computed
if (col.uidt === UITypes.Checkbox) { if (col.uidt === UITypes.Checkbox) {
return value ? GROUP_BY_VARS.TRUE : GROUP_BY_VARS.FALSE return value ? GROUP_BY_VARS.TRUE : GROUP_BY_VARS.FALSE
} }
// convert to JSON string if non-string value
if (value && typeof value === 'object') {
value = JSON.stringify(value)
}
return value ?? GROUP_BY_VARS.NULL return value ?? GROUP_BY_VARS.NULL
} }
@ -144,13 +150,13 @@ export const useViewGroupBy = (view: Ref<ViewType | undefined>, where?: Computed
const calculateNestedWhere = (nestedIn: GroupNestedIn[], existing = '') => { const calculateNestedWhere = (nestedIn: GroupNestedIn[], existing = '') => {
return nestedIn.reduce((acc, curr) => { return nestedIn.reduce((acc, curr) => {
if (curr.key === GROUP_BY_VARS.NULL) { if (curr.key === GROUP_BY_VARS.NULL) {
acc += `${acc.length ? '~and' : ''}(${curr.title},blank)` acc += `${acc.length ? '~and' : ''}(${curr.title},gb_null)`
} else if (curr.column_uidt === UITypes.Checkbox) { } else if (curr.column_uidt === UITypes.Checkbox) {
acc += `${acc.length ? '~and' : ''}(${curr.title},${curr.key === GROUP_BY_VARS.TRUE ? 'checked' : 'notchecked'})` acc += `${acc.length ? '~and' : ''}(${curr.title},${curr.key === GROUP_BY_VARS.TRUE ? 'checked' : 'notchecked'})`
} else if ([UITypes.Date, UITypes.DateTime].includes(curr.column_uidt as UITypes)) { } else if ([UITypes.Date, UITypes.DateTime].includes(curr.column_uidt as UITypes)) {
acc += `${acc.length ? '~and' : ''}(${curr.title},eq,exactDate,${curr.key})` acc += `${acc.length ? '~and' : ''}(${curr.title},eq,exactDate,${curr.key})`
} else { } else {
acc += `${acc.length ? '~and' : ''}(${curr.title},eq,${curr.key})` acc += `${acc.length ? '~and' : ''}(${curr.title},gb_eq,${curr.key})`
} }
return acc return acc
}, existing) }, existing)

56
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -35,7 +35,6 @@ import type {
SelectOption, SelectOption,
} from '~/models'; } from '~/models';
import type { SortType } from 'nocodb-sdk'; import type { SortType } from 'nocodb-sdk';
import generateBTLookupSelectQuery from '~/db/generateBTLookupSelectQuery';
import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2';
import genRollupSelectv2 from '~/db/genRollupSelectv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2';
import conditionV2 from '~/db/conditionV2'; import conditionV2 from '~/db/conditionV2';
@ -60,10 +59,13 @@ import { HANDLE_WEBHOOK } from '~/services/hook-handler.service';
import { import {
COMPARISON_OPS, COMPARISON_OPS,
COMPARISON_SUB_OPS, COMPARISON_SUB_OPS,
GROUPBY_COMPARISON_OPS,
IS_WITHIN_COMPARISON_SUB_OPS, IS_WITHIN_COMPARISON_SUB_OPS,
} from '~/utils/globals'; } from '~/utils/globals';
import { extractProps } from '~/helpers/extractProps'; import { extractProps } from '~/helpers/extractProps';
import { defaultLimitConfig } from '~/helpers/extractLimitAndOffset'; import { defaultLimitConfig } from '~/helpers/extractLimitAndOffset';
import generateLookupSelectQuery from '~/db/generateLookupSelectQuery';
import { getAliasGenerator } from '~/utils';
dayjs.extend(utc); dayjs.extend(utc);
@ -386,6 +388,7 @@ class BaseModelSqlv2 {
validateFormula: true, validateFormula: true,
}); });
} }
return data?.map((d) => { return data?.map((d) => {
d.__proto__ = proto; d.__proto__ = proto;
return d; return d;
@ -549,18 +552,32 @@ class BaseModelSqlv2 {
const selectors = []; const selectors = [];
const groupBySelectors = []; const groupBySelectors = [];
const getAlias = getAliasGenerator('__nc_gb');
await Promise.all( await Promise.all(
args.column_name.split(',').map(async (col) => { args.column_name.split(',').map(async (col) => {
const column = cols.find( let column = cols.find((c) => c.column_name === col || c.title === col);
(c) => c.column_name === col || c.title === col,
);
groupByColumns[column.id] = column;
if (!column) { if (!column) {
throw NcError.notFound('Column not found'); throw NcError.notFound('Column not found');
} }
// if qrCode or Barcode replace it with value column nd keep the alias
if ([UITypes.QrCode, UITypes.Barcode].includes(column.uidt))
column = new Column({
...(await column
.getColOptions<BarcodeColumn | QrCodeColumn>()
.then((col) => col.getValueColumn())),
title: column.title,
});
groupByColumns[column.id] = column;
switch (column.uidt) { switch (column.uidt) {
case UITypes.Attachment:
NcError.badRequest(
'Group by using attachment column is not supported',
);
break;
case UITypes.Links: case UITypes.Links:
case UITypes.Rollup: case UITypes.Rollup:
selectors.push( selectors.push(
@ -599,12 +616,14 @@ class BaseModelSqlv2 {
} }
break; break;
case UITypes.Lookup: case UITypes.Lookup:
case UITypes.LinkToAnotherRecord:
{ {
const _selectQb = await generateBTLookupSelectQuery({ const _selectQb = await generateLookupSelectQuery({
baseModelSqlv2: this, baseModelSqlv2: this,
column, column,
alias: null, alias: null,
model: this.model, model: this.model,
getAlias,
}); });
const selectQb = this.dbDriver.raw(`?? as ??`, [ const selectQb = this.dbDriver.raw(`?? as ??`, [
@ -695,6 +714,7 @@ class BaseModelSqlv2 {
qb.groupBy(...groupBySelectors); qb.groupBy(...groupBySelectors);
applyPaginate(qb, rest); applyPaginate(qb, rest);
return await this.execAndParse(qb); return await this.execAndParse(qb);
} }
@ -711,18 +731,34 @@ class BaseModelSqlv2 {
const selectors = []; const selectors = [];
const groupBySelectors = []; const groupBySelectors = [];
const getAlias = getAliasGenerator('__nc_gb');
// todo: refactor and avoid duplicate code
await this.model.getColumns().then((cols) => await this.model.getColumns().then((cols) =>
Promise.all( Promise.all(
args.column_name.split(',').map(async (col) => { args.column_name.split(',').map(async (col) => {
const column = cols.find( let column = cols.find(
(c) => c.column_name === col || c.title === col, (c) => c.column_name === col || c.title === col,
); );
if (!column) { if (!column) {
throw NcError.notFound('Column not found'); throw NcError.notFound('Column not found');
} }
// if qrCode or Barcode replace it with value column nd keep the alias
if ([UITypes.QrCode, UITypes.Barcode].includes(column.uidt))
column = new Column({
...(await column
.getColOptions<BarcodeColumn | QrCodeColumn>()
.then((col) => col.getValueColumn())),
title: column.title,
});
switch (column.uidt) { switch (column.uidt) {
case UITypes.Attachment:
NcError.badRequest(
'Group by using attachment column is not supported',
);
break;
case UITypes.Rollup: case UITypes.Rollup:
case UITypes.Links: case UITypes.Links:
selectors.push( selectors.push(
@ -764,12 +800,14 @@ class BaseModelSqlv2 {
break; break;
} }
case UITypes.Lookup: case UITypes.Lookup:
case UITypes.LinkToAnotherRecord:
{ {
const _selectQb = await generateBTLookupSelectQuery({ const _selectQb = await generateLookupSelectQuery({
baseModelSqlv2: this, baseModelSqlv2: this,
column, column,
alias: null, alias: null,
model: this.model, model: this.model,
getAlias,
}); });
const selectQb = this.dbDriver.raw(`?? as ??`, [ const selectQb = this.dbDriver.raw(`?? as ??`, [
@ -5247,7 +5285,7 @@ export function extractFilterFromXwhere(
// mark `op` and `sub_op` any for being assignable to parameter of type // mark `op` and `sub_op` any for being assignable to parameter of type
function validateFilterComparison(uidt: UITypes, op: any, sub_op?: any) { function validateFilterComparison(uidt: UITypes, op: any, sub_op?: any) {
if (!COMPARISON_OPS.includes(op)) { if (!COMPARISON_OPS.includes(op) && !GROUPBY_COMPARISON_OPS.includes(op)) {
NcError.badRequest(`${op} is not supported.`); NcError.badRequest(`${op} is not supported.`);
} }

40
packages/nocodb/src/db/conditionV2.ts

@ -8,11 +8,14 @@ import type Column from '~/models/Column';
import type LookupColumn from '~/models/LookupColumn'; import type LookupColumn from '~/models/LookupColumn';
import type RollupColumn from '~/models/RollupColumn'; import type RollupColumn from '~/models/RollupColumn';
import type FormulaColumn from '~/models/FormulaColumn'; import type FormulaColumn from '~/models/FormulaColumn';
import type { BarcodeColumn, QrCodeColumn } from '~/models';
import { NcError } from '~/helpers/catchError'; import { NcError } from '~/helpers/catchError';
import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2'; import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2';
import genRollupSelectv2 from '~/db/genRollupSelectv2'; import genRollupSelectv2 from '~/db/genRollupSelectv2';
import { sanitize } from '~/helpers/sqlSanitize'; import { sanitize } from '~/helpers/sqlSanitize';
import Filter from '~/models/Filter'; import Filter from '~/models/Filter';
import generateLookupSelectQuery from '~/db/generateLookupSelectQuery';
import { getAliasGenerator } from '~/utils';
// tod: tobe fixed // tod: tobe fixed
// extend(customParseFormat); // extend(customParseFormat);
@ -112,6 +115,43 @@ const parseConditionV2 = async (
}); });
}; };
} else { } else {
// handle group by filter separately,
// `gb_eq` is equivalent to `eq` but for lookup it compares on aggregated value returns in group by api
// aggregated value will be either json array or `___` separated string
// `gb_null` is equivalent to `blank` but for lookup it compares on aggregated value is null
if (
(filter.comparison_op as any) === 'gb_eq' ||
(filter.comparison_op as any) === 'gb_null'
) {
const column = await filter.getColumn();
if (
column.uidt === UITypes.Lookup ||
column.uidt === UITypes.LinkToAnotherRecord
) {
const model = await column.getModel();
const lkQb = await generateLookupSelectQuery({
baseModelSqlv2,
alias: alias,
model,
column,
getAlias: getAliasGenerator('__gb_filter_lk'),
});
return (qb) => {
if ((filter.comparison_op as any) === 'gb_eq')
qb.where(knex.raw('?', [filter.value]), lkQb.builder);
else qb.whereNull(knex.raw(lkQb.builder).wrap('(', ')'));
};
} else {
filter.comparison_op =
(filter.comparison_op as any) === 'gb_eq' ? 'eq' : 'blank';
// if qrCode or Barcode replace it with value column
if ([UITypes.QrCode, UITypes.Barcode].includes(column.uidt))
filter.fk_column_id = await column
.getColOptions<BarcodeColumn | QrCodeColumn>()
.then((col) => col.fk_column_id);
}
}
const column = await filter.getColumn(); const column = await filter.getColumn();
if (!column) { if (!column) {
if (throwErrorIfInvalid) { if (throwErrorIfInvalid) {

163
packages/nocodb/src/db/generateBTLookupSelectQuery.ts

@ -1,163 +0,0 @@
import { RelationTypes, UITypes } from 'nocodb-sdk';
import type LookupColumn from '../models/LookupColumn';
import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
import type {
Column,
FormulaColumn,
LinkToAnotherRecordColumn,
Model,
RollupColumn,
} from '~/models';
import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2';
import genRollupSelectv2 from '~/db/genRollupSelectv2';
import { NcError } from '~/helpers/catchError';
export default async function generateBTLookupSelectQuery({
column,
baseModelSqlv2,
alias,
model,
}: {
column: Column;
baseModelSqlv2: BaseModelSqlv2;
alias: string;
model: Model;
}): Promise<any> {
const knex = baseModelSqlv2.dbDriver;
const rootAlias = alias;
{
let aliasCount = 0,
selectQb;
const alias = `__nc_lk_${aliasCount++}`;
const lookup = await column.getColOptions<LookupColumn>();
{
const relationCol = await lookup.getRelationColumn();
const relation =
await relationCol.getColOptions<LinkToAnotherRecordColumn>();
// if not belongs to then throw error as we don't support
if (relation.type !== RelationTypes.BELONGS_TO)
NcError.badRequest('HasMany/ManyToMany lookup is not supported');
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb = knex(
`${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`,
).where(
`${alias}.${parentColumn.column_name}`,
knex.raw(`??`, [
`${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${
childColumn.column_name
}`,
]),
);
}
let lookupColumn = await lookup.getLookupColumn();
let prevAlias = alias;
while (lookupColumn.uidt === UITypes.Lookup) {
const nestedAlias = `__nc_lk_nested_${aliasCount++}`;
const nestedLookup = await lookupColumn.getColOptions<LookupColumn>();
const relationCol = await nestedLookup.getRelationColumn();
const relation =
await relationCol.getColOptions<LinkToAnotherRecordColumn>();
// if any of the relation in nested lookup is
// not belongs to then throw error as we don't support
if (relation.type !== RelationTypes.BELONGS_TO)
NcError.badRequest('HasMany/ManyToMany lookup is not supported');
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb.join(
`${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${nestedAlias}`,
`${nestedAlias}.${parentColumn.column_name}`,
`${prevAlias}.${childColumn.column_name}`,
);
lookupColumn = await nestedLookup.getLookupColumn();
prevAlias = nestedAlias;
}
switch (lookupColumn.uidt) {
case UITypes.Links:
case UITypes.Rollup:
{
const builder = (
await genRollupSelectv2({
baseModelSqlv2,
knex,
columnOptions:
(await lookupColumn.getColOptions()) as RollupColumn,
alias: prevAlias,
})
).builder;
selectQb.select(builder);
}
break;
case UITypes.LinkToAnotherRecord:
{
const nestedAlias = `__nc_sort${aliasCount++}`;
const relation =
await lookupColumn.getColOptions<LinkToAnotherRecordColumn>();
if (relation.type !== 'bt') return;
const colOptions =
(await column.getColOptions()) as LinkToAnotherRecordColumn;
const childColumn = await colOptions.getChildColumn();
const parentColumn = await colOptions.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb
.join(
`${baseModelSqlv2.getTnPath(
parentModel.table_name,
)} as ${nestedAlias}`,
`${nestedAlias}.${parentColumn.column_name}`,
`${prevAlias}.${childColumn.column_name}`,
)
.select(parentModel?.displayValue?.column_name);
}
break;
case UITypes.Formula:
{
const builder = (
await formulaQueryBuilderv2(
baseModelSqlv2,
(
await column.getColOptions<FormulaColumn>()
).formula,
null,
model,
column,
)
).builder;
selectQb.select(builder);
}
break;
default:
{
selectQb.select(`${prevAlias}.${lookupColumn.column_name}`);
}
break;
}
return { builder: selectQb };
}
}

399
packages/nocodb/src/db/generateLookupSelectQuery.ts

@ -0,0 +1,399 @@
import { RelationTypes, UITypes } from 'nocodb-sdk';
import type LookupColumn from '../models/LookupColumn';
import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
import type {
BarcodeColumn,
Column,
FormulaColumn,
LinksColumn,
LinkToAnotherRecordColumn,
QrCodeColumn,
RollupColumn,
} from '~/models';
import { Model } from '~/models';
import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2';
import genRollupSelectv2 from '~/db/genRollupSelectv2';
import { getAliasGenerator } from '~/utils';
import { NcError } from '~/helpers/catchError';
const LOOKUP_VAL_SEPARATOR = '___';
export async function getDisplayValueOfRefTable(
relationCol: Column<LinkToAnotherRecordColumn | LinksColumn>,
) {
return await relationCol
.getColOptions()
.then((colOpt) => colOpt.getRelatedTable())
.then((model) => model.getColumns())
.then((cols) => cols.find((col) => col.pv));
}
export default async function generateLookupSelectQuery({
column,
baseModelSqlv2,
alias,
model: _model,
getAlias = getAliasGenerator('__lk_slt_'),
}: {
column: Column;
baseModelSqlv2: BaseModelSqlv2;
alias: string;
model: Model;
getAlias?: ReturnType<typeof getAliasGenerator>;
}): Promise<any> {
const knex = baseModelSqlv2.dbDriver;
const rootAlias = alias;
{
let selectQb;
const alias = getAlias();
let lookupColOpt: LookupColumn;
let isBtLookup = true;
if (column.uidt === UITypes.Lookup) {
lookupColOpt = await column.getColOptions<LookupColumn>();
} else if (column.uidt !== UITypes.LinkToAnotherRecord) {
NcError.badRequest('Invalid column type');
}
await column.getColOptions<LookupColumn>();
{
const relationCol = lookupColOpt
? await lookupColOpt.getRelationColumn()
: column;
const relation =
await relationCol.getColOptions<LinkToAnotherRecordColumn>();
// if not belongs to then throw error as we don't support
if (relation.type === RelationTypes.BELONGS_TO) {
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb = knex(
`${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`,
).where(
`${alias}.${parentColumn.column_name}`,
knex.raw(`??`, [
`${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${
childColumn.column_name
}`,
]),
);
}
// if not belongs to then throw error as we don't support
else if (relation.type === RelationTypes.HAS_MANY) {
isBtLookup = false;
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb = knex(
`${baseModelSqlv2.getTnPath(childModel.table_name)} as ${alias}`,
).where(
`${alias}.${childColumn.column_name}`,
knex.raw(`??`, [
`${rootAlias || baseModelSqlv2.getTnPath(parentModel.table_name)}.${
parentColumn.column_name
}`,
]),
);
}
// if not belongs to then throw error as we don't support
else if (relation.type === RelationTypes.MANY_TO_MANY) {
isBtLookup = false;
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb = knex(
`${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`,
);
const mmTableAlias = getAlias();
const mmModel = await relation.getMMModel();
const mmChildCol = await relation.getMMChildColumn();
const mmParentCol = await relation.getMMParentColumn();
selectQb
.innerJoin(
baseModelSqlv2.getTnPath(mmModel.table_name, mmTableAlias),
knex.ref(`${mmTableAlias}.${mmParentCol.column_name}`),
'=',
knex.ref(`${alias}.${parentColumn.column_name}`),
)
.where(
knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`),
'=',
knex.ref(
`${
rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)
}.${childColumn.column_name}`,
),
);
}
}
let lookupColumn = lookupColOpt
? await lookupColOpt.getLookupColumn()
: await getDisplayValueOfRefTable(column);
// if lookup column is qr code or barcode extract the referencing column
if ([UITypes.QrCode, UITypes.Barcode].includes(lookupColumn.uidt)) {
lookupColumn = await lookupColumn
.getColOptions<BarcodeColumn | QrCodeColumn>()
.then((barcode) => barcode.getValueColumn());
}
let prevAlias = alias;
while (
lookupColumn.uidt === UITypes.Lookup ||
lookupColumn.uidt === UITypes.LinkToAnotherRecord
) {
const nestedAlias = getAlias();
let relationCol: Column<LinkToAnotherRecordColumn | LinksColumn>;
let nestedLookupColOpt: LookupColumn;
if (lookupColumn.uidt === UITypes.Lookup) {
nestedLookupColOpt = await lookupColumn.getColOptions<LookupColumn>();
relationCol = await nestedLookupColOpt.getRelationColumn();
} else {
relationCol = lookupColumn;
}
const relation =
await relationCol.getColOptions<LinkToAnotherRecordColumn>();
// if any of the relation in nested lookupColOpt is
// not belongs to then throw error as we don't support
if (relation.type === RelationTypes.BELONGS_TO) {
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb.join(
`${baseModelSqlv2.getTnPath(
parentModel.table_name,
)} as ${nestedAlias}`,
`${nestedAlias}.${parentColumn.column_name}`,
`${prevAlias}.${childColumn.column_name}`,
);
} else if (relation.type === RelationTypes.HAS_MANY) {
isBtLookup = false;
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb.join(
`${baseModelSqlv2.getTnPath(
childModel.table_name,
)} as ${nestedAlias}`,
`${nestedAlias}.${childColumn.column_name}`,
`${prevAlias}.${parentColumn.column_name}`,
);
} else if (relation.type === RelationTypes.MANY_TO_MANY) {
isBtLookup = false;
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
const mmTableAlias = getAlias();
const mmModel = await relation.getMMModel();
const mmChildCol = await relation.getMMChildColumn();
const mmParentCol = await relation.getMMParentColumn();
selectQb
.innerJoin(
baseModelSqlv2.getTnPath(mmModel.table_name, mmTableAlias),
knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`),
'=',
knex.ref(`${prevAlias}.${childColumn.column_name}`),
)
.innerJoin(
knex.raw('?? as ??', [
baseModelSqlv2.getTnPath(parentModel.table_name),
nestedAlias,
]),
knex.ref(`${mmTableAlias}.${mmParentCol.column_name}`),
'=',
knex.ref(`${nestedAlias}.${parentColumn.column_name}`),
)
.where(
knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`),
'=',
knex.ref(
`${alias || baseModelSqlv2.getTnPath(childModel.table_name)}.${
childColumn.column_name
}`,
),
);
}
if (lookupColumn.uidt === UITypes.Lookup)
lookupColumn = await nestedLookupColOpt.getLookupColumn();
else lookupColumn = await getDisplayValueOfRefTable(relationCol);
prevAlias = nestedAlias;
}
{
// get basemodel and model of lookup column
const model = await lookupColumn.getModel();
const baseModelSqlv2 = await Model.getBaseModelSQL({
model,
dbDriver: knex,
});
switch (lookupColumn.uidt) {
case UITypes.Attachment:
NcError.badRequest(
'Group by using attachment column is not supported',
);
break;
case UITypes.Links:
case UITypes.Rollup:
{
const builder = (
await genRollupSelectv2({
baseModelSqlv2,
knex,
columnOptions:
(await lookupColumn.getColOptions()) as RollupColumn,
alias: prevAlias,
})
).builder;
selectQb.select(builder);
}
break;
case UITypes.Formula:
{
const builder = (
await formulaQueryBuilderv2(
baseModelSqlv2,
(
await lookupColumn.getColOptions<FormulaColumn>()
).formula,
lookupColumn.title,
model,
lookupColumn,
await model.getAliasColMapping(),
prevAlias,
)
).builder;
selectQb.select(builder);
}
break;
case UITypes.DateTime:
{
await baseModelSqlv2.selectObject({
qb: selectQb,
columns: [lookupColumn],
alias: prevAlias,
});
}
break;
default:
{
selectQb.select(
`${prevAlias}.${lookupColumn.column_name} as ${lookupColumn.title}`,
);
}
break;
}
}
// if all relation are belongs to then we don't need to do the aggregation
if (isBtLookup) {
return {
builder: selectQb,
};
}
const subQueryAlias = getAlias();
if (baseModelSqlv2.isPg) {
// alternate approach with array_agg
return {
builder: knex
.select(knex.raw('json_agg(??)::text', [lookupColumn.title]))
.from(selectQb.as(subQueryAlias)),
};
/*
// alternate approach with array_agg
return {
builder: knex
.select(knex.raw('array_agg(??)', [lookupColumn.title]))
.from(selectQb),
};*/
// alternate approach with string aggregation
// return {
// builder: knex
// .select(
// knex.raw('STRING_AGG(??::text, ?)', [
// lookupColumn.title,
// LOOKUP_VAL_SEPARATOR,
// ]),
// )
// .from(selectQb.as(subQueryAlias)),
// };
} else if (baseModelSqlv2.isMySQL) {
return {
builder: knex
.select(
knex.raw('cast(JSON_ARRAYAGG(??) as NCHAR)', [lookupColumn.title]),
)
.from(selectQb.as(subQueryAlias)),
};
// return {
// builder: knex
// .select(
// knex.raw('GROUP_CONCAT(?? ORDER BY ?? ASC SEPARATOR ?)', [
// lookupColumn.title,
// lookupColumn.title,
// LOOKUP_VAL_SEPARATOR,
// ]),
// )
// .from(selectQb.as(subQueryAlias)),
// };
} else if (baseModelSqlv2.isSqlite) {
// ref: https://stackoverflow.com/questions/13382856/sqlite3-join-group-concat-using-distinct-with-custom-separator
// selectQb.orderBy(`${lookupColumn.title}`, 'asc');
return {
builder: knex
.select(
knex.raw(`group_concat(??, ?)`, [
lookupColumn.title,
LOOKUP_VAL_SEPARATOR,
]),
)
.from(selectQb.as(subQueryAlias)),
};
}
NcError.notImplemented('Database not supported Group by on Lookup');
}
}

5
packages/nocodb/src/utils/globals.ts

@ -171,6 +171,11 @@ export enum CacheDelDirection {
CHILD_TO_PARENT = 'CHILD_TO_PARENT', CHILD_TO_PARENT = 'CHILD_TO_PARENT',
} }
export const GROUPBY_COMPARISON_OPS = <const>[
// these are used for groupby
'gb_eq',
'gb_null',
];
export const COMPARISON_OPS = <const>[ export const COMPARISON_OPS = <const>[
'eq', 'eq',
'neq', 'neq',

10
packages/nocodb/tests/unit/rest/tests/groupby.test.ts

@ -299,7 +299,7 @@ function groupByTests() {
expect(response.body.list.length).to.equal(1); expect(response.body.list.length).to.equal(1);
}); });
it('Check One GroupBy Column with MM Lookup which is not supported', async function () { it('Check One GroupBy Column with MM Lookup which is supported', async function () {
await createLookupColumn(context, { await createLookupColumn(context, {
base: sakilaProject, base: sakilaProject,
title: 'ActorNames', title: 'ActorNames',
@ -308,15 +308,17 @@ function groupByTests() {
relatedTableColumnTitle: 'FirstName', relatedTableColumnTitle: 'FirstName',
}); });
const res = await request(context.app) const response = await request(context.app)
.get(`/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/groupby`) .get(`/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/groupby`)
.set('xc-auth', context.token) .set('xc-auth', context.token)
.query({ .query({
column_name: 'ActorNames', column_name: 'ActorNames',
}) })
.expect(400); .expect(200);
assert.match(res.body.msg, /not supported/); assert.match(response.body.list[1]['ActorNames'], /ADAM|ANNE/);
expect(+response.body.list[1]['count']).to.gt(0);
expect(response.body.list.length).to.equal(25);
}); });
it('Check One GroupBy Column with Formula and Formula referring another formula', async function () { it('Check One GroupBy Column with Formula and Formula referring another formula', async function () {

Loading…
Cancel
Save