Browse Source

Merge branch 'develop' into feat/yyyy-mm

pull/6870/head
Raju Udava 12 months ago
parent
commit
0f741a0864
  1. 3
      packages/nc-gui/components.d.ts
  2. 8
      packages/nc-gui/components/cell/Json.vue
  3. 12
      packages/nc-gui/components/cell/TextArea.vue
  4. 2
      packages/nc-gui/components/dashboard/TreeView/ProjectNode.vue
  5. 7
      packages/nc-gui/components/dlg/AirtableImport.vue
  6. 14
      packages/nc-gui/components/dlg/ColumnDuplicate.vue
  7. 14
      packages/nc-gui/components/dlg/QuickImport.vue
  8. 2
      packages/nc-gui/components/smartsheet/Cell.vue
  9. 44
      packages/nc-gui/components/smartsheet/column/SelectOptions.vue
  10. 11
      packages/nc-gui/components/smartsheet/details/Fields.vue
  11. 39
      packages/nc-gui/components/smartsheet/grid/GroupBy.vue
  12. 28
      packages/nc-gui/components/smartsheet/grid/GroupByLabel.vue
  13. 22
      packages/nc-gui/components/smartsheet/header/Menu.vue
  14. 50
      packages/nc-gui/components/smartsheet/toolbar/GroupByMenu.vue
  15. 85
      packages/nc-gui/components/template/Editor.vue
  16. 6
      packages/nc-gui/components/virtual-cell/Lookup.vue
  17. 2
      packages/nc-gui/components/virtual-cell/components/ListItems.vue
  18. 10
      packages/nc-gui/composables/useColumnCreateStore.ts
  19. 10
      packages/nc-gui/composables/useViewGroupBy.ts
  20. 22
      packages/nc-gui/utils/validation.ts
  21. 2
      packages/nc-lib-gui/package.json
  22. 2
      packages/noco-docs/docs/150.engineering/070.translation.md
  23. 2
      packages/nocodb-sdk/package.json
  24. 6
      packages/nocodb/package.json
  25. 39
      packages/nocodb/src/controllers/api-docs/api-docs.controller.ts
  26. 56
      packages/nocodb/src/db/BaseModelSqlv2.ts
  27. 42
      packages/nocodb/src/db/conditionV2.ts
  28. 163
      packages/nocodb/src/db/generateBTLookupSelectQuery.ts
  29. 399
      packages/nocodb/src/db/generateLookupSelectQuery.ts
  30. 5
      packages/nocodb/src/helpers/getAst.ts
  31. 7
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
  32. 20
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts
  33. 11
      packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts
  34. 30
      packages/nocodb/src/services/api-docs/api-docs.service.ts
  35. 2
      packages/nocodb/src/services/api-docs/swagger/templates/paths.ts
  36. 28
      packages/nocodb/src/services/api-docs/swaggerV2/getPaths.ts
  37. 29
      packages/nocodb/src/services/api-docs/swaggerV2/getSchemas.ts
  38. 67
      packages/nocodb/src/services/api-docs/swaggerV2/getSwaggerColumnMetas.ts
  39. 66
      packages/nocodb/src/services/api-docs/swaggerV2/getSwaggerJSONV2.ts
  40. 128
      packages/nocodb/src/services/api-docs/swaggerV2/swagger-base.json
  41. 10
      packages/nocodb/src/services/api-docs/swaggerV2/templates/headers.ts
  42. 237
      packages/nocodb/src/services/api-docs/swaggerV2/templates/params.ts
  43. 407
      packages/nocodb/src/services/api-docs/swaggerV2/templates/paths.ts
  44. 109
      packages/nocodb/src/services/api-docs/swaggerV2/templates/schemas.ts
  45. 3
      packages/nocodb/src/services/datas.service.ts
  46. 4
      packages/nocodb/src/services/tables.service.ts
  47. 5
      packages/nocodb/src/utils/globals.ts
  48. 10
      packages/nocodb/tests/unit/rest/tests/groupby.test.ts
  49. 491
      pnpm-lock.yaml
  50. 3
      renovate.json
  51. 6
      tests/playwright/pages/Dashboard/Import/ImportTemplate.ts

3
packages/nc-gui/components.d.ts vendored

@ -26,7 +26,6 @@ declare module '@vue/runtime-core' {
AConfigProvider: typeof import('ant-design-vue/es')['ConfigProvider']
ADatePicker: typeof import('ant-design-vue/es')['DatePicker']
ADivider: typeof import('ant-design-vue/es')['Divider']
ADrawer: typeof import('ant-design-vue/es')['Drawer']
ADropdown: typeof import('ant-design-vue/es')['Dropdown']
ADropdownButton: typeof import('ant-design-vue/es')['DropdownButton']
AEmpty: typeof import('ant-design-vue/es')['Empty']
@ -47,7 +46,6 @@ declare module '@vue/runtime-core' {
AMenu: typeof import('ant-design-vue/es')['Menu']
AMenuDivider: typeof import('ant-design-vue/es')['MenuDivider']
AMenuItem: typeof import('ant-design-vue/es')['MenuItem']
AMenuItemGroup: typeof import('ant-design-vue/es')['MenuItemGroup']
AModal: typeof import('ant-design-vue/es')['Modal']
APagination: typeof import('ant-design-vue/es')['Pagination']
APopover: typeof import('ant-design-vue/es')['Popover']
@ -64,7 +62,6 @@ declare module '@vue/runtime-core' {
ASubMenu: typeof import('ant-design-vue/es')['SubMenu']
ASwitch: typeof import('ant-design-vue/es')['Switch']
ATable: typeof import('ant-design-vue/es')['Table']
ATableColumn: typeof import('ant-design-vue/es')['TableColumn']
ATabPane: typeof import('ant-design-vue/es')['TabPane']
ATabs: typeof import('ant-design-vue/es')['Tabs']
ATag: typeof import('ant-design-vue/es')['Tag']

8
packages/nc-gui/components/cell/Json.vue

@ -72,13 +72,7 @@ const clear = () => {
const formatJson = (json: string) => {
try {
json = json
.trim()
.replace(/^\{\s*|\s*\}$/g, '')
.replace(/\n\s*/g, '')
json = `{${json}}`
return json
return JSON.stringify(JSON.parse(json))
} catch (e) {
console.log(e)
return json

12
packages/nc-gui/components/cell/TextArea.vue

@ -16,6 +16,7 @@ import {
const props = defineProps<{
modelValue?: string | number
isFocus?: boolean
virtual?: boolean
}>()
const emits = defineEmits(['update:modelValue'])
@ -65,6 +66,13 @@ onClickOutside(inputWrapperRef, (e) => {
isVisible.value = false
})
const onDblClick = () => {
if (!props.virtual) return
isVisible.value = true
editEnabled.value = true
}
</script>
<template>
@ -113,7 +121,9 @@ onClickOutside(inputWrapperRef, (e) => {
class="mr-7 nc-text-area-clamped-text"
:style="{
'word-break': 'break-word',
'white-space': 'pre-line',
}"
@click="onDblClick"
/>
<span v-else>{{ vModel }}</span>
@ -148,7 +158,7 @@ onClickOutside(inputWrapperRef, (e) => {
<a-textarea
ref="inputRef"
v-model:value="vModel"
class="p-1 !pt-1 !pr-3 !border-0 !border-r-0 !focus:outline-transparent nc-scrollbar-md !text-black"
class="p-1 !pt-1 !pr-3 !border-0 !border-r-0 !focus:outline-transparent nc-scrollbar-md !text-black !cursor-text"
:placeholder="$t('activity.enterText')"
:bordered="false"
:auto-size="{ minRows: 20, maxRows: 20 }"

2
packages/nc-gui/components/dashboard/TreeView/ProjectNode.vue

@ -514,7 +514,7 @@ const projectDelete = () => {
@click.stop="
() => {
$e('c:base:api-docs')
openLink(`/api/v1/db/meta/projects/${base.id}/swagger`, appInfo.ncSiteUrl)
openLink(`/api/v2/meta/bases/${base.id}/swagger`, appInfo.ncSiteUrl)
}
"
>

7
packages/nc-gui/components/dlg/AirtableImport.vue

@ -90,8 +90,10 @@ const onStatus = async (status: JobStatus, data?: any) => {
refreshCommandPalette()
// TODO: add tab of the first table
} else if (status === JobStatus.FAILED) {
await loadTables()
goBack.value = true
pushProgress(data.error.message, status)
refreshCommandPalette()
}
}
@ -115,7 +117,10 @@ const { validateInfos } = useForm(syncSource, validators)
const disableImportButton = computed(() => !syncSource.value.details.apiKey || !syncSource.value.details.syncSourceUrlOrId)
const isLoading = ref(false)
async function saveAndSync() {
isLoading.value = true
await createOrUpdate()
await sync()
}
@ -178,6 +183,7 @@ async function listenForUpdates(id?: string) {
}
} else {
listeningForUpdates.value = false
isLoading.value = false
}
},
)
@ -494,6 +500,7 @@ onMounted(async () => {
v-e="['c:sync-airtable:save-and-sync']"
type="primary"
class="nc-btn-airtable-import"
:loading="isLoading"
:disabled="disableImportButton"
@click="saveAndSync"
>

14
packages/nc-gui/components/dlg/ColumnDuplicate.vue

@ -17,10 +17,6 @@ const dialogShow = useVModel(props, 'modelValue', emit)
const { $e, $poller } = useNuxtApp()
const basesStore = useBases()
const { createProject: _createProject } = basesStore
const { activeTable: _activeTable } = storeToRefs(useTablesStore())
const reloadDataHook = inject(ReloadViewDataHookInj)
@ -101,7 +97,9 @@ onKeyStroke('Enter', () => {
}
})
const isEaster = ref(false)
defineExpose({
duplicate: _duplicate,
})
</script>
<template>
@ -118,11 +116,9 @@ const isEaster = ref(false)
@keydown.esc="dialogShow = false"
>
<div>
<div class="prose-xl font-bold self-center" @dblclick="isEaster = !isEaster">
{{ $t('general.duplicate') }} {{ $t('objects.column') }}
</div>
<div class="prose-xl font-bold self-center">{{ $t('general.duplicate') }} {{ $t('objects.column') }}</div>
<div class="mt-4">{{ $t('msg.warning.duplicateProject') }}</div>
<div class="mt-4">Are you sure you want to duplicate the field?</div>
<div class="prose-md self-center text-gray-500 mt-4">{{ $t('title.advancedSettings') }}</div>

14
packages/nc-gui/components/dlg/QuickImport.vue

@ -172,7 +172,9 @@ const disablePreImportButton = computed(() => {
}
})
const disableImportButton = computed(() => !templateEditorRef.value?.isValid)
const isError = ref(false)
const disableImportButton = computed(() => !templateEditorRef.value?.isValid || isError.value)
const disableFormatJsonButton = computed(() => !jsonEditorRef.value?.isValid)
@ -530,6 +532,14 @@ async function parseAndExtractData(val: UploadFile[] | ArrayBuffer | string) {
preImportLoading.value = false
}
}
const onError = () => {
isError.value = true
}
const onChange = () => {
isError.value = false
}
</script>
<template>
@ -558,6 +568,8 @@ async function parseAndExtractData(val: UploadFile[] | ArrayBuffer | string) {
:import-worker="importWorker"
class="nc-quick-import-template-editor"
@import="handleImport"
@error="onError"
@change="onChange"
/>
<a-tabs v-else v-model:activeKey="activeKey" hide-add type="editable-card" tab-position="top">

2
packages/nc-gui/components/smartsheet/Cell.vue

@ -216,7 +216,7 @@ onUnmounted(() => {
>
<template v-if="column">
<template v-if="intersected">
<LazyCellTextArea v-if="isTextArea(column)" v-model="vModel" />
<LazyCellTextArea v-if="isTextArea(column)" v-model="vModel" :virtual="props.virtual" />
<LazyCellGeoData v-else-if="isGeoData(column)" v-model="vModel" />
<LazyCellCheckbox v-else-if="isBoolean(column, abstractType)" v-model="vModel" />
<LazyCellAttachment v-else-if="isAttachment(column)" v-model="vModel" :row-index="props.rowIndex" />

44
packages/nc-gui/components/smartsheet/column/SelectOptions.vue

@ -29,7 +29,7 @@ const { optionsMagic: _optionsMagic } = useNocoEe()
const optionsWrapperDomRef = ref<HTMLElement>()
const options = ref<(Option & { status?: 'remove' })[]>([])
const options = ref<(Option & { status?: 'remove'; index?: number })[]>([])
const isAddingOption = ref(false)
@ -38,7 +38,7 @@ const OPTIONS_PAGE_COUNT = 20
const loadedOptionAnchor = ref(OPTIONS_PAGE_COUNT)
const isReverseLazyLoad = ref(false)
const renderedOptions = ref<(Option & { status?: 'remove' })[]>([])
const renderedOptions = ref<(Option & { status?: 'remove'; index?: number })[]>([])
const savedDefaultOption = ref<Option | null>(null)
const savedCdf = ref<string | null>(null)
@ -98,6 +98,12 @@ onMounted(() => {
options.value = vModel.value.colOptions.options
let indexCounter = 0
options.value.map((el) => {
el.index = indexCounter++
return el
})
loadedOptionAnchor.value = Math.min(loadedOptionAnchor.value, options.value.length)
renderedOptions.value = [...options.value].slice(0, loadedOptionAnchor.value)
@ -135,6 +141,7 @@ const addNewOption = () => {
const tempOption = {
title: '',
color: getNextColor(),
index: options.value.length,
}
options.value.push(tempOption)
@ -168,11 +175,30 @@ const addNewOption = () => {
// }
const syncOptions = () => {
vModel.value.colOptions.options = renderedOptions.value.filter((op) => op.status !== 'remove')
vModel.value.colOptions.options = options.value
.filter((op) => op.status !== 'remove')
.sort((a, b) => {
const renderA = renderedOptions.value.findIndex((el) => a.index !== undefined && el.index === a.index)
const renderB = renderedOptions.value.findIndex((el) => a.index !== undefined && el.index === b.index)
if (renderA === -1 || renderB === -1) return 0
return renderA - renderB
})
.map((op) => {
const { index: _i, status: _s, ...rest } = op
return rest
})
}
const removeRenderedOption = (index: number) => {
renderedOptions.value[index].status = 'remove'
const renderedOption = renderedOptions.value[index]
if (renderedOption.index === undefined || isNaN(renderedOption.index)) return
const option = options.value[renderedOption.index]
renderedOption.status = 'remove'
option.status = 'remove'
syncOptions()
const optionId = renderedOptions.value[index]?.id
@ -193,7 +219,15 @@ const optionChanged = (changedId: string) => {
}
const undoRemoveRenderedOption = (index: number) => {
renderedOptions.value[index].status = undefined
const renderedOption = renderedOptions.value[index]
if (renderedOption.index === undefined || isNaN(renderedOption.index)) return
const option = options.value[renderedOption.index]
renderedOption.status = undefined
option.status = undefined
syncOptions()
const optionId = renderedOptions.value[index]?.id

11
packages/nc-gui/components/smartsheet/details/Fields.vue

@ -1,7 +1,7 @@
<script setup lang="ts">
import { diff } from 'deep-object-diff'
import { message } from 'ant-design-vue'
import { UITypes, isSystemColumn } from 'nocodb-sdk'
import { UITypes, isSystemColumn, isVirtualCol } from 'nocodb-sdk'
import Draggable from 'vuedraggable'
import { onKeyDown, useMagicKeys } from '@vueuse/core'
import type { ColumnType, SelectOptionsType } from 'nocodb-sdk'
@ -774,8 +774,15 @@ const onFieldOptionUpdate = () => {
"
/>
<NcCheckbox v-else :disabled="true" class="opacity-0" :checked="true" />
<SmartsheetHeaderVirtualCellIcon
v-if="field && isVirtualCol(fieldState(field) || field)"
:column-meta="fieldState(field) || field"
:class="{
'text-brand-500': compareCols(field, activeField),
}"
/>
<SmartsheetHeaderCellIcon
v-if="field"
v-else
:column-meta="fieldState(field) || field"
:class="{
'text-brand-500': compareCols(field, activeField),

39
packages/nc-gui/components/smartsheet/grid/GroupBy.vue

@ -1,8 +1,10 @@
<script lang="ts" setup>
import tinycolor from 'tinycolor2'
import { UITypes } from 'nocodb-sdk'
import Table from './Table.vue'
import GroupBy from './GroupBy.vue'
import GroupByTable from './GroupByTable.vue'
import GroupByLabel from './GroupByLabel.vue'
import { GROUP_BY_VARS, computed, ref } from '#imports'
import type { Group, Row } from '#imports'
@ -134,6 +136,27 @@ const onScroll = (e: Event) => {
if (!vGroup.value.root) return
_scrollLeft.value = (e.target as HTMLElement).scrollLeft
}
// a method to parse group key if grouped column type is LTAR or Lookup
// in these 2 scenario it will return json array or `___` separated value
const parseKey = (group) => {
const key = group.key.toString()
// parse json array key if it's a lookup or link to another record
if ((key && group.column?.uidt === UITypes.Lookup) || group.column?.uidt === UITypes.LinkToAnotherRecord) {
try {
const parsedKey = JSON.parse(key)
return parsedKey
} catch {
// if parsing try to split it by `___` (for sqlite)
return key.split('___')
}
}
return [key]
}
const shouldRenderCell = (column) =>
[UITypes.Lookup, UITypes.Attachment, UITypes.Barcode, UITypes.QrCode, UITypes.Links].includes(column?.uidt)
</script>
<template>
@ -227,6 +250,15 @@ const onScroll = (e: Event) => {
</span>
</a-tag>
</template>
<div
v-else-if="!(grp.key in GROUP_BY_VARS.VAR_TITLES) && shouldRenderCell(grp.column)"
class="flex min-w-[100px] flex-wrap"
>
<template v-for="(val, ind) of parseKey(grp)" :key="ind">
<GroupByLabel v-if="val" :column="grp.column" :model-value="val" />
<span v-else class="text-gray-400">No mapped value</span>
</template>
</div>
<a-tag
v-else
:key="`panel-tag-${grp.column.id}-${grp.key}`"
@ -247,7 +279,12 @@ const onScroll = (e: Event) => {
'font-weight': 500,
}"
>
{{ grp.key in GROUP_BY_VARS.VAR_TITLES ? GROUP_BY_VARS.VAR_TITLES[grp.key] : grp.key }}
<template v-if="grp.key in GROUP_BY_VARS.VAR_TITLES">{{
GROUP_BY_VARS.VAR_TITLES[grp.key]
}}</template>
<template v-else>
{{ parseKey(grp)?.join(', ') }}
</template>
</span>
</a-tag>
</div>

28
packages/nc-gui/components/smartsheet/grid/GroupByLabel.vue

@ -0,0 +1,28 @@
<script setup lang="ts">
import type { ColumnType } from 'nocodb-sdk'
import { isVirtualCol } from 'nocodb-sdk'
defineProps<{
column: ColumnType
modelValue: any
}>()
provide(ReadonlyInj, true)
</script>
<template>
<div class="pointer-events-none">
<LazySmartsheetRow :row="{ row: { [column.title]: modelValue }, rowMeta: {} }">
<LazySmartsheetVirtualCell v-if="isVirtualCol(column)" :model-value="modelValue" class="!text-gray-600" :column="column" />
<LazySmartsheetCell
v-else
:model-value="modelValue"
class="!text-gray-600"
:column="column"
:edit-enabled="false"
:read-only="true"
/>
</LazySmartsheetRow>
</div>
</template>

22
packages/nc-gui/components/smartsheet/header/Menu.vue

@ -1,5 +1,5 @@
<script lang="ts" setup>
import type { ColumnReqType, ColumnType } from 'nocodb-sdk'
import type { ColumnReqType } from 'nocodb-sdk'
import { RelationTypes, UITypes, isLinksOrLTAR } from 'nocodb-sdk'
import {
ActiveViewInj,
@ -111,8 +111,8 @@ const sortByColumn = async (direction: 'asc' | 'desc') => {
}
const isDuplicateDlgOpen = ref(false)
const selectedColumnToDuplicate = ref<ColumnType>()
const selectedColumnExtra = ref<any>()
const duplicateDialogRef = ref<any>()
const duplicateVirtualColumn = async () => {
let columnCreatePayload = {}
@ -165,7 +165,7 @@ const duplicateVirtualColumn = async () => {
const openDuplicateDlg = async () => {
if (!column?.value) return
if (column.value.uidt && [UITypes.Formula, UITypes.Lookup, UITypes.Rollup].includes(column.value.uidt as UITypes)) {
if (column.value.uidt && [UITypes.Lookup, UITypes.Rollup].includes(column.value.uidt as UITypes)) {
duplicateVirtualColumn()
} else {
const gridViewColumnList = (await $api.dbViewColumn.list(view.value?.id as string)).list
@ -186,8 +186,15 @@ const openDuplicateDlg = async () => {
view_id: view.value!.id as string,
},
}
selectedColumnToDuplicate.value = column.value
isDuplicateDlgOpen.value = true
if (column.value.uidt === UITypes.Formula) {
nextTick(() => {
duplicateDialogRef?.value?.duplicate()
})
} else {
isDuplicateDlgOpen.value = true
}
isOpen.value = false
}
}
@ -373,9 +380,10 @@ const onInsertAfter = () => {
</a-dropdown>
<SmartsheetHeaderDeleteColumnModal v-model:visible="showDeleteColumnModal" />
<DlgColumnDuplicate
v-if="selectedColumnToDuplicate"
v-if="column"
ref="duplicateDialogRef"
v-model="isDuplicateDlgOpen"
:column="selectedColumnToDuplicate"
:column="column"
:extra="selectedColumnExtra"
/>
</template>

50
packages/nc-gui/components/smartsheet/toolbar/GroupByMenu.vue

@ -1,6 +1,6 @@
<script setup lang="ts">
import type { ColumnType, LinkToAnotherRecordType, LookupType } from 'nocodb-sdk'
import { RelationTypes, UITypes } from 'nocodb-sdk'
import { UITypes } from 'nocodb-sdk'
import {
ActiveViewInj,
IsLockedInj,
@ -15,20 +15,10 @@ import {
useNuxtApp,
useSmartsheetStoreOrThrow,
useViewColumnsOrThrow,
watch,
} from '#imports'
const groupingUidt = [
UITypes.SingleSelect,
UITypes.MultiSelect,
UITypes.Checkbox,
UITypes.Date,
UITypes.SingleLineText,
UITypes.Number,
UITypes.Rollup,
UITypes.Lookup,
UITypes.Links,
UITypes.Formula,
]
const excludedGroupingUidt = [UITypes.Attachment]
const meta = inject(MetaInj, ref())
const view = inject(ActiveViewInj, ref())
@ -62,16 +52,16 @@ const groupedByColumnIds = computed(() => groupBy.value.map((g) => g.fk_column_i
const { eventBus } = useSmartsheetStoreOrThrow()
const { isMobileMode } = useGlobal()
const btLookups = ref([])
const supportedLookups = ref([])
const fieldsToGroupBy = computed(() => {
const fields = meta.value?.columns || []
return fields.filter((field) => {
if (!groupingUidt.includes(field.uidt as UITypes)) return false
if (excludedGroupingUidt.includes(field.uidt as UITypes)) return false
if (field.uidt === UITypes.Lookup) {
return btLookups.value.includes(field.id)
return supportedLookups.value.includes(field.id)
}
return true
@ -161,25 +151,18 @@ watch(open, () => {
}
})
const loadBtLookups = async () => {
const loadAllowedLookups = async () => {
const filteredLookupCols = []
try {
for (const col of meta.value?.columns || []) {
if (col.uidt !== UITypes.Lookup) continue
let nextCol = col
let btLookup = true
// check all the relation of nested lookup columns is bt or not
// include the column only if all only if all relations are bt
while (btLookup && nextCol && nextCol.uidt === UITypes.Lookup) {
// check the lookup column is supported type or not
while (nextCol && nextCol.uidt === UITypes.Lookup) {
const lookupRelation = (await getMeta(nextCol.fk_model_id))?.columns?.find(
(c) => c.id === (nextCol.colOptions as LookupType).fk_relation_column_id,
)
if ((lookupRelation.colOptions as LinkToAnotherRecordType).type !== RelationTypes.BELONGS_TO) {
btLookup = false
continue
}
const relatedTableMeta = await getMeta((lookupRelation.colOptions as LinkToAnotherRecordType).fk_related_model_id)
@ -190,22 +173,25 @@ const loadBtLookups = async () => {
// if next column is same as root lookup column then break the loop
// since it's going to be a circular loop, and ignore the column
if (nextCol.id === col.id) {
btLookup = false
break
}
}
if (btLookup) filteredLookupCols.push(col.id)
if (nextCol.uidt !== UITypes.Attachment) filteredLookupCols.push(col.id)
}
btLookups.value = filteredLookupCols
supportedLookups.value = filteredLookupCols
} catch (e) {
console.error(e)
}
}
onMounted(async () => {
await loadBtLookups()
await loadAllowedLookups()
})
watch(meta, async () => {
await loadAllowedLookups()
})
</script>
@ -242,9 +228,7 @@ onMounted(async () => {
<LazySmartsheetToolbarFieldListAutoCompleteDropdown
v-model="group.fk_column_id"
class="caption nc-sort-field-select"
:columns="
fieldsToGroupBy.filter((f) => (f.id && !groupedByColumnIds.includes(f.id)) || f.id === group.fk_column_id)
"
:columns="fieldsToGroupBy"
:allow-empty="true"
@change="saveGroupBy"
@click.stop

85
packages/nc-gui/components/template/Editor.vue

@ -31,12 +31,13 @@ import {
useI18n,
useNuxtApp,
useTabs,
validateTableName,
} from '#imports'
const { quickImportType, baseTemplate, importData, importColumns, importDataOnly, maxRowsToParse, sourceId, importWorker } =
defineProps<Props>()
const emit = defineEmits(['import'])
const emit = defineEmits(['import', 'error', 'change'])
dayjs.extend(utc)
@ -92,6 +93,8 @@ const importingTips = ref<Record<string, string>>({})
const checkAllRecord = ref<boolean[]>([])
const formError = ref()
const uiTypeOptions = ref<Option[]>(
(Object.keys(UITypes) as (keyof typeof UITypes)[])
.filter(
@ -121,11 +124,14 @@ const data = reactive<{
const validators = computed(() =>
data.tables.reduce<Record<string, [ReturnType<typeof fieldRequiredValidator>]>>((acc: Record<string, any>, table, tableIdx) => {
acc[`tables.${tableIdx}.table_name`] = [fieldRequiredValidator()]
acc[`tables.${tableIdx}.table_name`] = [validateTableName]
hasSelectColumn.value[tableIdx] = false
table.columns?.forEach((column, columnIdx) => {
acc[`tables.${tableIdx}.columns.${columnIdx}.column_name`] = [fieldRequiredValidator(), fieldLengthValidator()]
acc[`tables.${tableIdx}.columns.${columnIdx}.column_name`] = [
fieldRequiredValidator(),
fieldLengthValidator(base.value?.sources?.[0].type || ClientType.MYSQL),
]
acc[`tables.${tableIdx}.columns.${columnIdx}.uidt`] = [fieldRequiredValidator()]
if (isSelect(column)) {
hasSelectColumn.value[tableIdx] = true
@ -136,10 +142,12 @@ const validators = computed(() =>
}, {}),
)
const { validate, validateInfos } = useForm(data, validators)
const { validate, validateInfos, modelRef } = useForm(data, validators)
const isValid = ref(!importDataOnly)
const formRef = ref()
watch(
() => srcDestMapping.value,
() => {
@ -671,6 +679,39 @@ function handleUIDTChange(column, table) {
])
}
}
const setErrorState = (errorsFields: any[]) => {
const errorMap: any = {}
for (const error of errorsFields) {
errorMap[error.name] = error.errors
}
formError.value = errorMap
}
watch(formRef, () => {
setTimeout(async () => {
try {
await validate()
emit('change')
formError.value = null
} catch (e: any) {
emit('error', e)
setErrorState(e.errorFields)
}
}, 500)
})
watch(modelRef, async () => {
try {
await validate()
emit('change')
formError.value = null
} catch (e: any) {
emit('error', e)
setErrorState(e.errorFields)
}
})
</script>
<template>
@ -691,7 +732,7 @@ function handleUIDTChange(column, table) {
<a-collapse v-if="data.tables && data.tables.length" v-model:activeKey="expansionPanel" class="template-collapse" accordion>
<a-collapse-panel v-for="(table, tableIdx) of data.tables" :key="tableIdx">
<template #header>
<span class="font-weight-bold text-lg flex items-center gap-2">
<span class="font-weight-bold text-lg flex items-center gap-2 truncate">
<component :is="iconMap.table" class="text-primary" />
{{ table.table_name }}
</span>
@ -766,7 +807,7 @@ function handleUIDTChange(column, table) {
</a-card>
<a-card v-else>
<a-form :model="data" name="template-editor-form" @keydown.enter="emit('import')">
<a-form ref="formRef" :model="data" name="template-editor-form" @keydown.enter="emit('import')">
<p v-if="data.tables && quickImportType === 'excel'" class="text-center">
{{ data.tables.length }} sheet{{ data.tables.length > 1 ? 's' : '' }}
available for import
@ -780,22 +821,24 @@ function handleUIDTChange(column, table) {
>
<a-collapse-panel v-for="(table, tableIdx) of data.tables" :key="tableIdx">
<template #header>
<a-form-item v-if="editableTn[tableIdx]" v-bind="validateInfos[`tables.${tableIdx}.table_name`]" no-style>
<a-input
v-model:value.lazy="table.table_name"
class="max-w-xs font-weight-bold text-lg"
size="large"
hide-details
:bordered="false"
@click.stop
@blur="handleEditableTnChange(tableIdx)"
@keydown.enter="handleEditableTnChange(tableIdx)"
/>
<a-form-item v-bind="validateInfos[`tables.${tableIdx}.table_name`]" no-style>
<div class="flex flex-col w-full">
<a-input
v-model:value="table.table_name"
class="font-weight-bold text-lg"
size="large"
hide-details
:bordered="false"
@click.stop
@blur="handleEditableTnChange(tableIdx)"
@keydown.enter="handleEditableTnChange(tableIdx)"
@dblclick="setEditableTn(tableIdx, true)"
/>
<div v-if="formError?.[`tables.${tableIdx}.table_name`]" class="text-red-500 ml-3">
{{ formError?.[`tables.${tableIdx}.table_name`].join('\n') }}
</div>
</div>
</a-form-item>
<span v-else class="font-weight-bold text-lg flex items-center gap-2" @click="setEditableTn(tableIdx, true)">
<component :is="iconMap.table" class="text-primary" />
{{ table.table_name }}
</span>
</template>
<template #extra>

6
packages/nc-gui/components/virtual-cell/Lookup.vue

@ -98,7 +98,7 @@ const { showEditNonEditableFieldWarning, showClearNonEditableFieldWarning, activ
<template>
<div
class="h-full w-full"
class="h-full w-full nc-lookup-cell"
:style="{ height: rowHeight && rowHeight !== 1 ? `${rowHeight * 2}rem` : `2.85rem` }"
@dblclick="activateShowEditNonEditableFieldWarning"
>
@ -206,4 +206,8 @@ const { showEditNonEditableFieldWarning, showClearNonEditableFieldWarning, activ
@apply bg-gray-200;
}
}
.nc-lookup-cell .nc-text-area-clamped-text {
@apply !mr-1;
}
</style>

2
packages/nc-gui/components/virtual-cell/components/ListItems.vue

@ -1,5 +1,5 @@
<script lang="ts" setup>
import { RelationTypes, UITypes, isLinksOrLTAR, isSystemColumn } from 'nocodb-sdk'
import { RelationTypes, isLinksOrLTAR, isSystemColumn } from 'nocodb-sdk'
import type { ColumnType, LinkToAnotherRecordType } from 'nocodb-sdk'
import InboxIcon from '~icons/nc-icons/inbox'
import {

10
packages/nc-gui/composables/useColumnCreateStore.ts

@ -40,6 +40,8 @@ const [useProvideColumnCreateStore, useColumnCreateStore] = createInjectionState
const { sqlUis } = storeToRefs(baseStore)
const { bases } = storeToRefs(useBases())
const { $api } = useNuxtApp()
const { getMeta } = useMetas()
@ -64,6 +66,12 @@ const [useProvideColumnCreateStore, useColumnCreateStore] = createInjectionState
isXcdbBaseFunc(meta.value?.source_id ? meta.value?.source_id : Object.keys(sqlUis.value)[0]),
)
const source = computed(() =>
meta.value && meta.value.source_id && meta.value.base_id
? bases.value.get(meta.value?.base_id as string)?.sources?.find((s) => s.id === meta.value!.source_id)
: undefined,
)
const idType = null
const additionalValidations = ref<ValidationsObj>({})
@ -128,7 +136,7 @@ const [useProvideColumnCreateStore, useColumnCreateStore] = createInjectionState
})
},
},
fieldLengthValidator(),
fieldLengthValidator(source.value?.type || ClientType.MYSQL),
],
uidt: [
{

10
packages/nc-gui/composables/useViewGroupBy.ts

@ -89,6 +89,12 @@ export const useViewGroupBy = (view: Ref<ViewType | undefined>, where?: Computed
if (col.uidt === UITypes.Checkbox) {
return value ? GROUP_BY_VARS.TRUE : GROUP_BY_VARS.FALSE
}
// convert to JSON string if non-string value
if (value && typeof value === 'object') {
value = JSON.stringify(value)
}
return value ?? GROUP_BY_VARS.NULL
}
@ -144,13 +150,13 @@ export const useViewGroupBy = (view: Ref<ViewType | undefined>, where?: Computed
const calculateNestedWhere = (nestedIn: GroupNestedIn[], existing = '') => {
return nestedIn.reduce((acc, curr) => {
if (curr.key === GROUP_BY_VARS.NULL) {
acc += `${acc.length ? '~and' : ''}(${curr.title},blank)`
acc += `${acc.length ? '~and' : ''}(${curr.title},gb_null)`
} else if (curr.column_uidt === UITypes.Checkbox) {
acc += `${acc.length ? '~and' : ''}(${curr.title},${curr.key === GROUP_BY_VARS.TRUE ? 'checked' : 'notchecked'})`
} else if ([UITypes.Date, UITypes.DateTime].includes(curr.column_uidt as UITypes)) {
acc += `${acc.length ? '~and' : ''}(${curr.title},eq,exactDate,${curr.key})`
} else {
acc += `${acc.length ? '~and' : ''}(${curr.title},eq,${curr.key})`
acc += `${acc.length ? '~and' : ''}(${curr.title},gb_eq,${curr.key})`
}
return acc
}, existing)

22
packages/nc-gui/utils/validation.ts

@ -13,6 +13,10 @@ export const validateTableName = {
return reject(new Error(t('msg.error.tableNameRequired')))
}
if (value.length > 52) {
return reject(new Error(t('msg.error.columnNameExceedsCharacters', { value: 52 })))
}
// exclude . / \
// rest all characters allowed
// https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/acreldb/n0rfg6x1shw0ppn1cwhco6yn09f7.htm#:~:text=By%20default%2C%20MySQL%20encloses%20column,not%20truncate%20a%20longer%20name.
@ -98,17 +102,21 @@ export const fieldRequiredValidator = () => {
}
}
export const fieldLengthValidator = () => {
export const fieldLengthValidator = (sqlClientType: string) => {
return {
validator: (rule: any, value: any) => {
const { t } = getI18n().global
// mysql allows 64 characters for column_name
// postgres allows 59 characters for column_name
// mssql allows 128 characters for column_name
// sqlite allows any number of characters for column_name
// We allow 255 for all databases, truncate will be handled by backend for column_name
const fieldLengthLimit = 255
// no limit for sqlite but set as 255
let fieldLengthLimit = 255
if (sqlClientType === 'mysql2' || sqlClientType === 'mysql') {
fieldLengthLimit = 64
} else if (sqlClientType === 'pg') {
fieldLengthLimit = 59
} else if (sqlClientType === 'mssql') {
fieldLengthLimit = 128
}
return new Promise((resolve, reject) => {
if (value?.length > fieldLengthLimit) {

2
packages/nc-lib-gui/package.json

@ -1,6 +1,6 @@
{
"name": "nc-lib-gui",
"version": "0.202.5",
"version": "0.202.7",
"description": "NocoDB GUI",
"author": {
"name": "NocoDB",

2
packages/noco-docs/docs/150.engineering/070.translation.md

@ -9,7 +9,7 @@ tags: ['Engineering']
## How to add / edit translations ?
### Using Github
### Using GitHub
- For English, make changes directly to [en.json](https://github.com/nocodb/nocodb/blob/develop/packages/nc-gui/lang/en.json) & commit to `develop`
- For any other language, use `crowdin` option.

2
packages/nocodb-sdk/package.json

@ -1,6 +1,6 @@
{
"name": "nocodb-sdk",
"version": "0.202.5",
"version": "0.202.7",
"description": "NocoDB SDK",
"main": "build/main/index.js",
"typings": "build/main/index.d.ts",

6
packages/nocodb/package.json

@ -1,6 +1,6 @@
{
"name": "nocodb",
"version": "0.202.5",
"version": "0.202.7",
"description": "NocoDB Backend",
"main": "dist/bundle.js",
"author": {
@ -131,7 +131,7 @@
"mysql2": "^3.2.0",
"nanoid": "^3.1.20",
"nc-help": "0.3.1",
"nc-lib-gui": "0.202.5",
"nc-lib-gui": "0.202.7",
"nc-plugin": "^0.1.3",
"ncp": "^2.0.0",
"nestjs-kafka": "^1.0.6",
@ -144,7 +144,7 @@
"p-queue": "^6.6.2",
"papaparse": "^5.4.0",
"parse-database-url": "^0.3.0",
"passport": "^0.4.1",
"passport": "^0.6.0",
"passport-auth-token": "^1.0.1",
"passport-custom": "^1.1.1",
"passport-github": "^1.1.0",

39
packages/nocodb/src/controllers/api-docs/api-docs.controller.ts

@ -18,10 +18,7 @@ import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard';
export class ApiDocsController {
constructor(private readonly apiDocsService: ApiDocsService) {}
@Get([
'/api/v1/db/meta/projects/:baseId/swagger.json',
'/api/v2/meta/bases/:baseId/swagger.json',
])
@Get(['/api/v1/db/meta/projects/:baseId/swagger.json'])
@UseGuards(MetaApiLimiterGuard, GlobalGuard)
@Acl('swaggerJson')
async swaggerJson(@Param('baseId') baseId: string, @Request() req) {
@ -33,21 +30,39 @@ export class ApiDocsController {
return swagger;
}
@Get([
'/api/v2/meta/bases/:baseId/swagger',
'/api/v1/db/meta/projects/:baseId/swagger',
])
@Get(['/api/v2/meta/bases/:baseId/swagger.json'])
@UseGuards(MetaApiLimiterGuard, GlobalGuard)
@Acl('swaggerJson')
async swaggerJsonV2(@Param('baseId') baseId: string, @Request() req) {
const swagger = await this.apiDocsService.swaggerJsonV2({
baseId: baseId,
siteUrl: req.ncSiteUrl,
});
return swagger;
}
@Get(['/api/v1/db/meta/projects/:baseId/swagger'])
@UseGuards(PublicApiLimiterGuard)
swaggerHtml(@Param('baseId') baseId: string, @Response() res) {
res.send(getSwaggerHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' }));
}
@UseGuards(PublicApiLimiterGuard)
@Get([
'/api/v1/db/meta/projects/:baseId/redoc',
'/api/v2/meta/bases/:baseId/redoc',
])
@Get(['/api/v1/db/meta/projects/:baseId/redoc'])
redocHtml(@Param('baseId') baseId: string, @Response() res) {
res.send(getRedocHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' }));
}
@Get(['/api/v2/meta/bases/:baseId/swagger'])
@UseGuards(PublicApiLimiterGuard)
swaggerHtmlV2(@Param('baseId') baseId: string, @Response() res) {
res.send(getSwaggerHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' }));
}
@UseGuards(PublicApiLimiterGuard)
@Get(['/api/v2/meta/bases/:baseId/redoc'])
redocHtmlV2(@Param('baseId') baseId: string, @Response() res) {
res.send(getRedocHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' }));
}
}

56
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -35,7 +35,6 @@ import type {
SelectOption,
} from '~/models';
import type { SortType } from 'nocodb-sdk';
import generateBTLookupSelectQuery from '~/db/generateBTLookupSelectQuery';
import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2';
import genRollupSelectv2 from '~/db/genRollupSelectv2';
import conditionV2 from '~/db/conditionV2';
@ -60,10 +59,13 @@ import { HANDLE_WEBHOOK } from '~/services/hook-handler.service';
import {
COMPARISON_OPS,
COMPARISON_SUB_OPS,
GROUPBY_COMPARISON_OPS,
IS_WITHIN_COMPARISON_SUB_OPS,
} from '~/utils/globals';
import { extractProps } from '~/helpers/extractProps';
import { defaultLimitConfig } from '~/helpers/extractLimitAndOffset';
import generateLookupSelectQuery from '~/db/generateLookupSelectQuery';
import { getAliasGenerator } from '~/utils';
dayjs.extend(utc);
@ -386,6 +388,7 @@ class BaseModelSqlv2 {
validateFormula: true,
});
}
return data?.map((d) => {
d.__proto__ = proto;
return d;
@ -549,18 +552,32 @@ class BaseModelSqlv2 {
const selectors = [];
const groupBySelectors = [];
const getAlias = getAliasGenerator('__nc_gb');
await Promise.all(
args.column_name.split(',').map(async (col) => {
const column = cols.find(
(c) => c.column_name === col || c.title === col,
);
groupByColumns[column.id] = column;
let column = cols.find((c) => c.column_name === col || c.title === col);
if (!column) {
throw NcError.notFound('Column not found');
}
// if qrCode or Barcode replace it with value column nd keep the alias
if ([UITypes.QrCode, UITypes.Barcode].includes(column.uidt))
column = new Column({
...(await column
.getColOptions<BarcodeColumn | QrCodeColumn>()
.then((col) => col.getValueColumn())),
title: column.title,
});
groupByColumns[column.id] = column;
switch (column.uidt) {
case UITypes.Attachment:
NcError.badRequest(
'Group by using attachment column is not supported',
);
break;
case UITypes.Links:
case UITypes.Rollup:
selectors.push(
@ -599,12 +616,14 @@ class BaseModelSqlv2 {
}
break;
case UITypes.Lookup:
case UITypes.LinkToAnotherRecord:
{
const _selectQb = await generateBTLookupSelectQuery({
const _selectQb = await generateLookupSelectQuery({
baseModelSqlv2: this,
column,
alias: null,
model: this.model,
getAlias,
});
const selectQb = this.dbDriver.raw(`?? as ??`, [
@ -695,6 +714,7 @@ class BaseModelSqlv2 {
qb.groupBy(...groupBySelectors);
applyPaginate(qb, rest);
return await this.execAndParse(qb);
}
@ -711,18 +731,34 @@ class BaseModelSqlv2 {
const selectors = [];
const groupBySelectors = [];
const getAlias = getAliasGenerator('__nc_gb');
// todo: refactor and avoid duplicate code
await this.model.getColumns().then((cols) =>
Promise.all(
args.column_name.split(',').map(async (col) => {
const column = cols.find(
let column = cols.find(
(c) => c.column_name === col || c.title === col,
);
if (!column) {
throw NcError.notFound('Column not found');
}
// if qrCode or Barcode replace it with value column nd keep the alias
if ([UITypes.QrCode, UITypes.Barcode].includes(column.uidt))
column = new Column({
...(await column
.getColOptions<BarcodeColumn | QrCodeColumn>()
.then((col) => col.getValueColumn())),
title: column.title,
});
switch (column.uidt) {
case UITypes.Attachment:
NcError.badRequest(
'Group by using attachment column is not supported',
);
break;
case UITypes.Rollup:
case UITypes.Links:
selectors.push(
@ -764,12 +800,14 @@ class BaseModelSqlv2 {
break;
}
case UITypes.Lookup:
case UITypes.LinkToAnotherRecord:
{
const _selectQb = await generateBTLookupSelectQuery({
const _selectQb = await generateLookupSelectQuery({
baseModelSqlv2: this,
column,
alias: null,
model: this.model,
getAlias,
});
const selectQb = this.dbDriver.raw(`?? as ??`, [
@ -5258,7 +5296,7 @@ export function extractFilterFromXwhere(
// mark `op` and `sub_op` any for being assignable to parameter of type
function validateFilterComparison(uidt: UITypes, op: any, sub_op?: any) {
if (!COMPARISON_OPS.includes(op)) {
if (!COMPARISON_OPS.includes(op) && !GROUPBY_COMPARISON_OPS.includes(op)) {
NcError.badRequest(`${op} is not supported.`);
}

42
packages/nocodb/src/db/conditionV2.ts

@ -8,11 +8,14 @@ import type Column from '~/models/Column';
import type LookupColumn from '~/models/LookupColumn';
import type RollupColumn from '~/models/RollupColumn';
import type FormulaColumn from '~/models/FormulaColumn';
import type { BarcodeColumn, QrCodeColumn } from '~/models';
import { NcError } from '~/helpers/catchError';
import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2';
import genRollupSelectv2 from '~/db/genRollupSelectv2';
import { sanitize } from '~/helpers/sqlSanitize';
import Filter from '~/models/Filter';
import generateLookupSelectQuery from '~/db/generateLookupSelectQuery';
import { getAliasGenerator } from '~/utils';
// tod: tobe fixed
// extend(customParseFormat);
@ -112,6 +115,43 @@ const parseConditionV2 = async (
});
};
} else {
// handle group by filter separately,
// `gb_eq` is equivalent to `eq` but for lookup it compares on aggregated value returns in group by api
// aggregated value will be either json array or `___` separated string
// `gb_null` is equivalent to `blank` but for lookup it compares on aggregated value is null
if (
(filter.comparison_op as any) === 'gb_eq' ||
(filter.comparison_op as any) === 'gb_null'
) {
const column = await filter.getColumn();
if (
column.uidt === UITypes.Lookup ||
column.uidt === UITypes.LinkToAnotherRecord
) {
const model = await column.getModel();
const lkQb = await generateLookupSelectQuery({
baseModelSqlv2,
alias: alias,
model,
column,
getAlias: getAliasGenerator('__gb_filter_lk'),
});
return (qb) => {
if ((filter.comparison_op as any) === 'gb_eq')
qb.where(knex.raw('?', [filter.value]), lkQb.builder);
else qb.whereNull(knex.raw(lkQb.builder).wrap('(', ')'));
};
} else {
filter.comparison_op =
(filter.comparison_op as any) === 'gb_eq' ? 'eq' : 'blank';
// if qrCode or Barcode replace it with value column
if ([UITypes.QrCode, UITypes.Barcode].includes(column.uidt))
filter.fk_column_id = await column
.getColOptions<BarcodeColumn | QrCodeColumn>()
.then((col) => col.fk_column_id);
}
}
const column = await filter.getColumn();
if (!column) {
if (throwErrorIfInvalid) {
@ -342,7 +382,7 @@ const parseConditionV2 = async (
return (qbP: Knex.QueryBuilder) => {
if (filter.comparison_op in negatedMapping)
qbP.where((qb) =>
qbP
qb
.whereNotIn(childColumn.column_name, selectQb)
.orWhereNull(childColumn.column_name),
);

163
packages/nocodb/src/db/generateBTLookupSelectQuery.ts

@ -1,163 +0,0 @@
import { RelationTypes, UITypes } from 'nocodb-sdk';
import type LookupColumn from '../models/LookupColumn';
import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
import type {
Column,
FormulaColumn,
LinkToAnotherRecordColumn,
Model,
RollupColumn,
} from '~/models';
import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2';
import genRollupSelectv2 from '~/db/genRollupSelectv2';
import { NcError } from '~/helpers/catchError';
export default async function generateBTLookupSelectQuery({
column,
baseModelSqlv2,
alias,
model,
}: {
column: Column;
baseModelSqlv2: BaseModelSqlv2;
alias: string;
model: Model;
}): Promise<any> {
const knex = baseModelSqlv2.dbDriver;
const rootAlias = alias;
{
let aliasCount = 0,
selectQb;
const alias = `__nc_lk_${aliasCount++}`;
const lookup = await column.getColOptions<LookupColumn>();
{
const relationCol = await lookup.getRelationColumn();
const relation =
await relationCol.getColOptions<LinkToAnotherRecordColumn>();
// if not belongs to then throw error as we don't support
if (relation.type !== RelationTypes.BELONGS_TO)
NcError.badRequest('HasMany/ManyToMany lookup is not supported');
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb = knex(
`${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`,
).where(
`${alias}.${parentColumn.column_name}`,
knex.raw(`??`, [
`${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${
childColumn.column_name
}`,
]),
);
}
let lookupColumn = await lookup.getLookupColumn();
let prevAlias = alias;
while (lookupColumn.uidt === UITypes.Lookup) {
const nestedAlias = `__nc_lk_nested_${aliasCount++}`;
const nestedLookup = await lookupColumn.getColOptions<LookupColumn>();
const relationCol = await nestedLookup.getRelationColumn();
const relation =
await relationCol.getColOptions<LinkToAnotherRecordColumn>();
// if any of the relation in nested lookup is
// not belongs to then throw error as we don't support
if (relation.type !== RelationTypes.BELONGS_TO)
NcError.badRequest('HasMany/ManyToMany lookup is not supported');
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb.join(
`${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${nestedAlias}`,
`${nestedAlias}.${parentColumn.column_name}`,
`${prevAlias}.${childColumn.column_name}`,
);
lookupColumn = await nestedLookup.getLookupColumn();
prevAlias = nestedAlias;
}
switch (lookupColumn.uidt) {
case UITypes.Links:
case UITypes.Rollup:
{
const builder = (
await genRollupSelectv2({
baseModelSqlv2,
knex,
columnOptions:
(await lookupColumn.getColOptions()) as RollupColumn,
alias: prevAlias,
})
).builder;
selectQb.select(builder);
}
break;
case UITypes.LinkToAnotherRecord:
{
const nestedAlias = `__nc_sort${aliasCount++}`;
const relation =
await lookupColumn.getColOptions<LinkToAnotherRecordColumn>();
if (relation.type !== 'bt') return;
const colOptions =
(await column.getColOptions()) as LinkToAnotherRecordColumn;
const childColumn = await colOptions.getChildColumn();
const parentColumn = await colOptions.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb
.join(
`${baseModelSqlv2.getTnPath(
parentModel.table_name,
)} as ${nestedAlias}`,
`${nestedAlias}.${parentColumn.column_name}`,
`${prevAlias}.${childColumn.column_name}`,
)
.select(parentModel?.displayValue?.column_name);
}
break;
case UITypes.Formula:
{
const builder = (
await formulaQueryBuilderv2(
baseModelSqlv2,
(
await column.getColOptions<FormulaColumn>()
).formula,
null,
model,
column,
)
).builder;
selectQb.select(builder);
}
break;
default:
{
selectQb.select(`${prevAlias}.${lookupColumn.column_name}`);
}
break;
}
return { builder: selectQb };
}
}

399
packages/nocodb/src/db/generateLookupSelectQuery.ts

@ -0,0 +1,399 @@
import { RelationTypes, UITypes } from 'nocodb-sdk';
import type LookupColumn from '../models/LookupColumn';
import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
import type {
BarcodeColumn,
Column,
FormulaColumn,
LinksColumn,
LinkToAnotherRecordColumn,
QrCodeColumn,
RollupColumn,
} from '~/models';
import { Model } from '~/models';
import formulaQueryBuilderv2 from '~/db/formulav2/formulaQueryBuilderv2';
import genRollupSelectv2 from '~/db/genRollupSelectv2';
import { getAliasGenerator } from '~/utils';
import { NcError } from '~/helpers/catchError';
const LOOKUP_VAL_SEPARATOR = '___';
export async function getDisplayValueOfRefTable(
relationCol: Column<LinkToAnotherRecordColumn | LinksColumn>,
) {
return await relationCol
.getColOptions()
.then((colOpt) => colOpt.getRelatedTable())
.then((model) => model.getColumns())
.then((cols) => cols.find((col) => col.pv));
}
export default async function generateLookupSelectQuery({
column,
baseModelSqlv2,
alias,
model: _model,
getAlias = getAliasGenerator('__lk_slt_'),
}: {
column: Column;
baseModelSqlv2: BaseModelSqlv2;
alias: string;
model: Model;
getAlias?: ReturnType<typeof getAliasGenerator>;
}): Promise<any> {
const knex = baseModelSqlv2.dbDriver;
const rootAlias = alias;
{
let selectQb;
const alias = getAlias();
let lookupColOpt: LookupColumn;
let isBtLookup = true;
if (column.uidt === UITypes.Lookup) {
lookupColOpt = await column.getColOptions<LookupColumn>();
} else if (column.uidt !== UITypes.LinkToAnotherRecord) {
NcError.badRequest('Invalid column type');
}
await column.getColOptions<LookupColumn>();
{
const relationCol = lookupColOpt
? await lookupColOpt.getRelationColumn()
: column;
const relation =
await relationCol.getColOptions<LinkToAnotherRecordColumn>();
// if not belongs to then throw error as we don't support
if (relation.type === RelationTypes.BELONGS_TO) {
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb = knex(
`${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`,
).where(
`${alias}.${parentColumn.column_name}`,
knex.raw(`??`, [
`${rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)}.${
childColumn.column_name
}`,
]),
);
}
// if not belongs to then throw error as we don't support
else if (relation.type === RelationTypes.HAS_MANY) {
isBtLookup = false;
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb = knex(
`${baseModelSqlv2.getTnPath(childModel.table_name)} as ${alias}`,
).where(
`${alias}.${childColumn.column_name}`,
knex.raw(`??`, [
`${rootAlias || baseModelSqlv2.getTnPath(parentModel.table_name)}.${
parentColumn.column_name
}`,
]),
);
}
// if not belongs to then throw error as we don't support
else if (relation.type === RelationTypes.MANY_TO_MANY) {
isBtLookup = false;
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb = knex(
`${baseModelSqlv2.getTnPath(parentModel.table_name)} as ${alias}`,
);
const mmTableAlias = getAlias();
const mmModel = await relation.getMMModel();
const mmChildCol = await relation.getMMChildColumn();
const mmParentCol = await relation.getMMParentColumn();
selectQb
.innerJoin(
baseModelSqlv2.getTnPath(mmModel.table_name, mmTableAlias),
knex.ref(`${mmTableAlias}.${mmParentCol.column_name}`),
'=',
knex.ref(`${alias}.${parentColumn.column_name}`),
)
.where(
knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`),
'=',
knex.ref(
`${
rootAlias || baseModelSqlv2.getTnPath(childModel.table_name)
}.${childColumn.column_name}`,
),
);
}
}
let lookupColumn = lookupColOpt
? await lookupColOpt.getLookupColumn()
: await getDisplayValueOfRefTable(column);
// if lookup column is qr code or barcode extract the referencing column
if ([UITypes.QrCode, UITypes.Barcode].includes(lookupColumn.uidt)) {
lookupColumn = await lookupColumn
.getColOptions<BarcodeColumn | QrCodeColumn>()
.then((barcode) => barcode.getValueColumn());
}
let prevAlias = alias;
while (
lookupColumn.uidt === UITypes.Lookup ||
lookupColumn.uidt === UITypes.LinkToAnotherRecord
) {
const nestedAlias = getAlias();
let relationCol: Column<LinkToAnotherRecordColumn | LinksColumn>;
let nestedLookupColOpt: LookupColumn;
if (lookupColumn.uidt === UITypes.Lookup) {
nestedLookupColOpt = await lookupColumn.getColOptions<LookupColumn>();
relationCol = await nestedLookupColOpt.getRelationColumn();
} else {
relationCol = lookupColumn;
}
const relation =
await relationCol.getColOptions<LinkToAnotherRecordColumn>();
// if any of the relation in nested lookupColOpt is
// not belongs to then throw error as we don't support
if (relation.type === RelationTypes.BELONGS_TO) {
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb.join(
`${baseModelSqlv2.getTnPath(
parentModel.table_name,
)} as ${nestedAlias}`,
`${nestedAlias}.${parentColumn.column_name}`,
`${prevAlias}.${childColumn.column_name}`,
);
} else if (relation.type === RelationTypes.HAS_MANY) {
isBtLookup = false;
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
selectQb.join(
`${baseModelSqlv2.getTnPath(
childModel.table_name,
)} as ${nestedAlias}`,
`${nestedAlias}.${childColumn.column_name}`,
`${prevAlias}.${parentColumn.column_name}`,
);
} else if (relation.type === RelationTypes.MANY_TO_MANY) {
isBtLookup = false;
const childColumn = await relation.getChildColumn();
const parentColumn = await relation.getParentColumn();
const childModel = await childColumn.getModel();
await childModel.getColumns();
const parentModel = await parentColumn.getModel();
await parentModel.getColumns();
const mmTableAlias = getAlias();
const mmModel = await relation.getMMModel();
const mmChildCol = await relation.getMMChildColumn();
const mmParentCol = await relation.getMMParentColumn();
selectQb
.innerJoin(
baseModelSqlv2.getTnPath(mmModel.table_name, mmTableAlias),
knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`),
'=',
knex.ref(`${prevAlias}.${childColumn.column_name}`),
)
.innerJoin(
knex.raw('?? as ??', [
baseModelSqlv2.getTnPath(parentModel.table_name),
nestedAlias,
]),
knex.ref(`${mmTableAlias}.${mmParentCol.column_name}`),
'=',
knex.ref(`${nestedAlias}.${parentColumn.column_name}`),
)
.where(
knex.ref(`${mmTableAlias}.${mmChildCol.column_name}`),
'=',
knex.ref(
`${alias || baseModelSqlv2.getTnPath(childModel.table_name)}.${
childColumn.column_name
}`,
),
);
}
if (lookupColumn.uidt === UITypes.Lookup)
lookupColumn = await nestedLookupColOpt.getLookupColumn();
else lookupColumn = await getDisplayValueOfRefTable(relationCol);
prevAlias = nestedAlias;
}
{
// get basemodel and model of lookup column
const model = await lookupColumn.getModel();
const baseModelSqlv2 = await Model.getBaseModelSQL({
model,
dbDriver: knex,
});
switch (lookupColumn.uidt) {
case UITypes.Attachment:
NcError.badRequest(
'Group by using attachment column is not supported',
);
break;
case UITypes.Links:
case UITypes.Rollup:
{
const builder = (
await genRollupSelectv2({
baseModelSqlv2,
knex,
columnOptions:
(await lookupColumn.getColOptions()) as RollupColumn,
alias: prevAlias,
})
).builder;
selectQb.select(builder);
}
break;
case UITypes.Formula:
{
const builder = (
await formulaQueryBuilderv2(
baseModelSqlv2,
(
await lookupColumn.getColOptions<FormulaColumn>()
).formula,
lookupColumn.title,
model,
lookupColumn,
await model.getAliasColMapping(),
prevAlias,
)
).builder;
selectQb.select(builder);
}
break;
case UITypes.DateTime:
{
await baseModelSqlv2.selectObject({
qb: selectQb,
columns: [lookupColumn],
alias: prevAlias,
});
}
break;
default:
{
selectQb.select(
`${prevAlias}.${lookupColumn.column_name} as ${lookupColumn.title}`,
);
}
break;
}
}
// if all relation are belongs to then we don't need to do the aggregation
if (isBtLookup) {
return {
builder: selectQb,
};
}
const subQueryAlias = getAlias();
if (baseModelSqlv2.isPg) {
// alternate approach with array_agg
return {
builder: knex
.select(knex.raw('json_agg(??)::text', [lookupColumn.title]))
.from(selectQb.as(subQueryAlias)),
};
/*
// alternate approach with array_agg
return {
builder: knex
.select(knex.raw('array_agg(??)', [lookupColumn.title]))
.from(selectQb),
};*/
// alternate approach with string aggregation
// return {
// builder: knex
// .select(
// knex.raw('STRING_AGG(??::text, ?)', [
// lookupColumn.title,
// LOOKUP_VAL_SEPARATOR,
// ]),
// )
// .from(selectQb.as(subQueryAlias)),
// };
} else if (baseModelSqlv2.isMySQL) {
return {
builder: knex
.select(
knex.raw('cast(JSON_ARRAYAGG(??) as NCHAR)', [lookupColumn.title]),
)
.from(selectQb.as(subQueryAlias)),
};
// return {
// builder: knex
// .select(
// knex.raw('GROUP_CONCAT(?? ORDER BY ?? ASC SEPARATOR ?)', [
// lookupColumn.title,
// lookupColumn.title,
// LOOKUP_VAL_SEPARATOR,
// ]),
// )
// .from(selectQb.as(subQueryAlias)),
// };
} else if (baseModelSqlv2.isSqlite) {
// ref: https://stackoverflow.com/questions/13382856/sqlite3-join-group-concat-using-distinct-with-custom-separator
// selectQb.orderBy(`${lookupColumn.title}`, 'asc');
return {
builder: knex
.select(
knex.raw(`group_concat(??, ?)`, [
lookupColumn.title,
LOOKUP_VAL_SEPARATOR,
]),
)
.from(selectQb.as(subQueryAlias)),
};
}
NcError.notImplemented('Database not supported Group by on Lookup');
}
}

5
packages/nocodb/src/helpers/getAst.ts

@ -68,7 +68,10 @@ const getAst = async ({
fields = Array.isArray(fields) ? fields : fields.split(',');
if (throwErrorIfInvalidParams) {
const colAliasMap = await model.getColAliasMapping();
const invalidFields = fields.filter((f) => !colAliasMap[f]);
const aliasColMap = await model.getAliasColMapping();
const invalidFields = fields.filter(
(f) => !colAliasMap[f] && !aliasColMap[f],
);
if (invalidFields.length) {
NcError.unprocessableEntity(
`Following fields are invalid: ${invalidFields.join(', ')}`,

7
packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@ -2529,6 +2529,13 @@ export class AtImportProcessor {
await generateMigrationStats(aTblSchema);
}
} catch (e) {
// delete tables that were created
for (const table of ncSchema.tables) {
await this.tablesService.tableDelete({
tableId: table.id,
user: syncDB.user,
});
}
if (e.message) {
this.telemetryService.sendEvent({
evt_type: 'a:airtable-import:error',

20
packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@ -3,7 +3,7 @@ import { Process, Processor } from '@nestjs/bull';
import { Job } from 'bull';
import papaparse from 'papaparse';
import debug from 'debug';
import { isLinksOrLTAR } from 'nocodb-sdk';
import { isLinksOrLTAR, isVirtualCol } from 'nocodb-sdk';
import { Base, Column, Model, Source } from '~/models';
import { BasesService } from '~/services/bases.service';
import {
@ -373,14 +373,16 @@ export class DuplicateProcessor {
});
// update cdf
await this.columnsService.columnUpdate({
columnId: findWithIdentifier(idMap, sourceColumn.id),
column: {
...destColumn,
cdf: oldCdf,
},
user: req.user,
});
if (!isVirtualCol(destColumn)) {
await this.columnsService.columnUpdate({
columnId: findWithIdentifier(idMap, sourceColumn.id),
column: {
...destColumn,
cdf: oldCdf,
},
user: req.user,
});
}
this.debugLog(`job completed for ${job.id} (${JobTypes.DuplicateModel})`);

11
packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@ -129,6 +129,17 @@ export class ExportService {
}
break;
case 'formula':
// rewrite formula_raw with aliases
column.colOptions['formula_raw'] = column.colOptions[k].replace(
/\{\{.*?\}\}/gm,
(match) => {
const col = model.columns.find(
(c) => c.id === match.slice(2, -2),
);
return `{${col?.title}}`;
},
);
column.colOptions[k] = column.colOptions[k].replace(
/(?<=\{\{).*?(?=\}\})/gm,
(match) => idMap.get(match),

30
packages/nocodb/src/services/api-docs/api-docs.service.ts

@ -1,5 +1,6 @@
import { Injectable } from '@nestjs/common';
import getSwaggerJSON from './swagger/getSwaggerJSON';
import getSwaggerJSONV2 from './swaggerV2/getSwaggerJSONV2';
import { NcError } from '~/helpers/catchError';
import { Base, Model } from '~/models';
@ -32,6 +33,35 @@ export class ApiDocsService {
},
] as any;
return swagger;
}
async swaggerJsonV2(param: { baseId: string; siteUrl: string }) {
const base = await Base.get(param.baseId);
if (!base) NcError.notFound();
const models = await Model.list({
base_id: param.baseId,
source_id: null,
});
const swagger = await getSwaggerJSONV2(base, models);
swagger.servers = [
{
url: param.siteUrl,
},
{
url: '{customUrl}',
variables: {
customUrl: {
default: param.siteUrl,
description: 'Provide custom nocodb app base url',
},
},
},
] as any;
return swagger;
}
}

2
packages/nocodb/src/services/api-docs/swagger/templates/paths.ts

@ -667,6 +667,6 @@ function getPaginatedResponseType(type: string) {
},
};
}
function isRelationExist(columns: SwaggerColumn[]) {
export function isRelationExist(columns: SwaggerColumn[]) {
return columns.some((c) => isLinksOrLTAR(c.column) && !c.column.system);
}

28
packages/nocodb/src/services/api-docs/swaggerV2/getPaths.ts

@ -0,0 +1,28 @@
import { getModelPaths } from './templates/paths';
import type { Model } from '~/models';
import type { SwaggerColumn } from './getSwaggerColumnMetas';
import type { SwaggerView } from './getSwaggerJSONV2';
import Noco from '~/Noco';
export default async function getPaths(
{
model,
columns,
views,
}: {
model: Model;
columns: SwaggerColumn[];
views: SwaggerView[];
},
_ncMeta = Noco.ncMeta,
) {
const swaggerPaths = await getModelPaths({
tableName: model.title,
tableId: model.id,
views,
type: model.type,
columns,
});
return swaggerPaths;
}

29
packages/nocodb/src/services/api-docs/swaggerV2/getSchemas.ts

@ -0,0 +1,29 @@
import { getModelSchemas } from './templates/schemas';
import type { Base, Model } from '~/models';
import type { SwaggerColumn } from './getSwaggerColumnMetas';
import type { SwaggerView } from './getSwaggerJSONV2';
import Noco from '~/Noco';
export default async function getSchemas(
{
base,
model,
columns,
}: {
base: Base;
model: Model;
columns: SwaggerColumn[];
views: SwaggerView[];
},
_ncMeta = Noco.ncMeta,
) {
const swaggerSchemas = getModelSchemas({
tableName: model.title,
orgs: 'v1',
baseName: base.title,
columns,
});
return swaggerSchemas;
}

67
packages/nocodb/src/services/api-docs/swaggerV2/getSwaggerColumnMetas.ts

@ -0,0 +1,67 @@
import { UITypes } from 'nocodb-sdk';
import type { Base, Column, LinkToAnotherRecordColumn } from '~/models';
import SwaggerTypes from '~/db/sql-mgr/code/routers/xc-ts/SwaggerTypes';
import Noco from '~/Noco';
export default async (
columns: Column[],
base: Base,
ncMeta = Noco.ncMeta,
): Promise<SwaggerColumn[]> => {
const dbType = await base.getBases().then((b) => b?.[0]?.type);
return Promise.all(
columns.map(async (c) => {
const field: SwaggerColumn = {
title: c.title,
type: 'object',
virtual: true,
column: c,
};
switch (c.uidt) {
case UITypes.LinkToAnotherRecord:
{
const colOpt = await c.getColOptions<LinkToAnotherRecordColumn>(
ncMeta,
);
if (colOpt) {
const relTable = await colOpt.getRelatedTable(ncMeta);
field.type = undefined;
field.$ref = `#/components/schemas/${relTable.title}Request`;
}
}
break;
case UITypes.Formula:
case UITypes.Lookup:
field.type = 'object';
break;
case UITypes.Rollup:
case UITypes.Links:
field.type = 'number';
break;
case UITypes.Attachment:
field.type = 'array';
field.items = {
$ref: `#/components/schemas/Attachment`,
};
break;
default:
field.virtual = false;
SwaggerTypes.setSwaggerType(c, field, dbType);
break;
}
return field;
}),
);
};
export interface SwaggerColumn {
type: any;
title: string;
description?: string;
virtual?: boolean;
$ref?: any;
column: Column;
items?: any;
}

66
packages/nocodb/src/services/api-docs/swaggerV2/getSwaggerJSONV2.ts

@ -0,0 +1,66 @@
import { ViewTypes } from 'nocodb-sdk';
import swaggerBase from './swagger-base.json';
import getPaths from './getPaths';
import getSchemas from './getSchemas';
import getSwaggerColumnMetas from './getSwaggerColumnMetas';
import type {
Base,
FormViewColumn,
GalleryViewColumn,
GridViewColumn,
Model,
View,
} from '~/models';
import Noco from '~/Noco';
export default async function getSwaggerJSONV2(
base: Base,
models: Model[],
ncMeta = Noco.ncMeta,
) {
// base swagger object
const swaggerObj = {
...swaggerBase,
paths: {},
components: {
...swaggerBase.components,
schemas: { ...swaggerBase.components.schemas },
},
};
// iterate and populate swagger schema and path for models and views
for (const model of models) {
let paths = {};
const columns = await getSwaggerColumnMetas(
await model.getColumns(ncMeta),
base,
ncMeta,
);
const views: SwaggerView[] = [];
for (const view of (await model.getViews(false, ncMeta)) || []) {
if (view.type !== ViewTypes.GRID) continue;
views.push({
view,
columns: await view.getColumns(ncMeta),
});
}
// skip mm tables
if (!model.mm) paths = await getPaths({ model, columns, views }, ncMeta);
const schemas = await getSchemas({ base, model, columns, views }, ncMeta);
Object.assign(swaggerObj.paths, paths);
Object.assign(swaggerObj.components.schemas, schemas);
}
return swaggerObj;
}
export interface SwaggerView {
view: View;
columns: Array<GridViewColumn | GalleryViewColumn | FormViewColumn>;
}

128
packages/nocodb/src/services/api-docs/swaggerV2/swagger-base.json

@ -0,0 +1,128 @@
{
"openapi": "3.0.0",
"info": {
"title": "nocodb",
"version": "2.0"
},
"servers": [
{
"url": "http://localhost:8080"
}
],
"paths": {
},
"components": {
"schemas": {
"Paginated": {
"title": "Paginated",
"type": "object",
"properties": {
"pageSize": {
"type": "integer"
},
"totalRows": {
"type": "integer"
},
"isFirstPage": {
"type": "boolean"
},
"isLastPage": {
"type": "boolean"
},
"page": {
"type": "number"
}
}
},
"Attachment": {
"title": "Attachment",
"type": "object",
"properties": {
"mimetype": {
"type": "string"
},
"size": {
"type": "integer"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
},
"icon": {
"type": "string"
}
}
},
"Groupby": {
"title": "Groupby",
"type": "object",
"properties": {
"count": {
"type": "number",
"description": "count"
},
"column_name": {
"type": "string",
"description": "the value of the given column"
}
}
}
},
"securitySchemes": {
"xcAuth": {
"type": "apiKey",
"in": "header",
"name": "xc-auth",
"description": "JWT access token"
},
"xcToken": {
"type": "apiKey",
"in": "header",
"name": "xc-token",
"description": "API token"
}
},
"responses": {
"BadRequest": {
"description": "BadReqeust",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"msg": {
"type": "string",
"x-stoplight": {
"id": "p9mk4oi0hbihm"
},
"example": "BadRequest [Error]: <ERROR MESSAGE>"
}
},
"required": [
"msg"
]
},
"examples": {
"Example 1": {
"value": {
"msg": "BadRequest [Error]: <ERROR MESSAGE>"
}
}
}
}
},
"headers": {}
}
}
},
"security": [
{
"xcAuth": []
},
{
"xcToken": []
}
]
}

10
packages/nocodb/src/services/api-docs/swaggerV2/templates/headers.ts

@ -0,0 +1,10 @@
export const csvExportResponseHeader = {
'nc-export-offset': {
schema: {
type: 'integer',
},
description:
'Offset of next set of data which will be helpful if there is large amount of data. It will returns `-1` if all set of data exported.',
example: '1000',
},
};

237
packages/nocodb/src/services/api-docs/swaggerV2/templates/params.ts

@ -0,0 +1,237 @@
import { isLinksOrLTAR, RelationTypes, UITypes } from 'nocodb-sdk';
import type { LinkToAnotherRecordColumn } from '~/models';
import type { SwaggerColumn } from '../getSwaggerColumnMetas';
import type { SwaggerView } from '~/services/api-docs/swaggerV2/getSwaggerJSONV2';
export const recordIdParam = {
schema: {
type: 'string',
},
name: 'recordId',
in: 'path',
required: true,
example: 1,
description:
'Primary key of the record you want to read. If the table have composite primary key then combine them by using `___` and pass it as primary key.',
};
export const fieldsParam = {
schema: {
type: 'string',
},
in: 'query',
name: 'fields',
description:
'Array of field names or comma separated filed names to include in the response objects. In array syntax pass it like `fields[]=field1&fields[]=field2` or alternately `fields=field1,field2`.',
};
export const sortParam = {
schema: {
type: 'string',
},
in: 'query',
name: 'sort',
description:
'Comma separated field names to sort rows, rows will sort in ascending order based on provided columns. To sort in descending order provide `-` prefix along with column name, like `-field`. Example : `sort=field1,-field2`',
};
export const whereParam = {
schema: {
type: 'string',
},
in: 'query',
name: 'where',
description:
'This can be used for filtering rows, which accepts complicated where conditions. For more info visit [here](https://docs.nocodb.com/developer-resources/rest-apis#comparison-operators). Example : `where=(field1,eq,value)`',
};
export const limitParam = {
schema: {
type: 'number',
minimum: 1,
},
in: 'query',
name: 'limit',
description:
'The `limit` parameter used for pagination, the response collection size depends on limit value with default value `25` and maximum value `1000`, which can be overridden by environment variables `DB_QUERY_LIMIT_DEFAULT` and `DB_QUERY_LIMIT_MAX` respectively.',
example: 25,
};
export const offsetParam = {
schema: {
type: 'number',
minimum: 0,
},
in: 'query',
name: 'offset',
description:
'The `offset` parameter used for pagination, the value helps to select collection from a certain index.',
example: 0,
};
export const shuffleParam = {
schema: {
type: 'number',
minimum: 0,
maximum: 1,
},
in: 'query',
name: 'shuffle',
description:
'The `shuffle` parameter used for pagination, the response will be shuffled if it is set to 1.',
example: 0,
};
export const columnNameQueryParam = {
schema: {
type: 'string',
},
in: 'query',
name: 'column_name',
description:
'Column name of the column you want to group by, eg. `column_name=column1`',
};
export const linkFieldNameParam = (columns: SwaggerColumn[]) => {
const linkColumnIds = [];
const description = [
'**Links Field Identifier** corresponding to the relation field `Links` established between tables.\n\nLink Columns:',
];
for (const { column } of columns) {
if (!isLinksOrLTAR(column) || column.system) continue;
linkColumnIds.push(column.id);
description.push(`* ${column.id} - ${column.title}`);
}
return {
schema: {
type: 'string',
enum: linkColumnIds,
},
name: 'linkFieldId',
in: 'path',
required: true,
description: description.join('\n'),
};
};
export const viewIdParams = (views: SwaggerView[]) => {
const viewIds = [];
const description = [
'Allows you to fetch records that are currently visible within a specific view.\n\nViews:',
];
for (const { view } of views) {
viewIds.push(view.id);
description.push(
`* ${view.id} - ${view.is_default ? 'Default view' : view.title}`,
);
}
return {
schema: {
type: 'string',
enum: viewIds,
},
description: description.join('\n'),
name: 'viewId',
in: 'query',
required: false,
};
};
export const referencedRowIdParam = {
schema: {
type: 'string',
},
name: 'refRowId',
in: 'path',
required: true,
};
export const exportTypeParam = {
schema: {
type: 'string',
enum: ['csv', 'excel'],
},
name: 'type',
in: 'path',
required: true,
};
export const csvExportOffsetParam = {
schema: {
type: 'number',
minimum: 0,
},
in: 'query',
name: 'offset',
description:
'Helps to start export from a certain index. You can get the next set of data offset from previous response header named `nc-export-offset`.',
example: 0,
};
export const nestedWhereParam = (colName) => ({
schema: {
type: 'string',
},
in: 'query',
name: `nested[${colName}][where]`,
description: `This can be used for filtering rows in nested column \`${colName}\`, which accepts complicated where conditions. For more info visit [here](https://docs.nocodb.com/developer-resources/rest-apis#comparison-operators). Example : \`nested[${colName}][where]=(field1,eq,value)\``,
});
export const nestedFieldParam = (colName) => ({
schema: {
type: 'string',
},
in: 'query',
name: `nested[${colName}][fields]`,
description: `Array of field names or comma separated filed names to include in the in nested column \`${colName}\` result. In array syntax pass it like \`fields[]=field1&fields[]=field2.\`. Example : \`nested[${colName}][fields]=field1,field2\``,
});
export const nestedSortParam = (colName) => ({
schema: {
type: 'string',
},
in: 'query',
name: `nested[${colName}][sort]`,
description: `Comma separated field names to sort rows in nested column \`${colName}\` rows, it will sort in ascending order based on provided columns. To sort in descending order provide \`-\` prefix along with column name, like \`-field\`. Example : \`nested[${colName}][sort]=field1,-field2\``,
});
export const nestedLimitParam = (colName) => ({
schema: {
type: 'number',
minimum: 1,
},
in: 'query',
name: `nested[${colName}][limit]`,
description: `The \`limit\` parameter used for pagination of nested \`${colName}\` rows, the response collection size depends on limit value and default value is \`25\`.`,
example: '25',
});
export const nestedOffsetParam = (colName) => ({
schema: {
type: 'number',
minimum: 0,
},
in: 'query',
name: `nested[${colName}][offset]`,
description: `The \`offset\` parameter used for pagination of nested \`${colName}\` rows, the value helps to select collection from a certain index.`,
example: 0,
});
export const getNestedParams = async (
columns: SwaggerColumn[],
): Promise<any[]> => {
return await columns.reduce(async (paramsArr, { column }) => {
if (column.uidt === UITypes.LinkToAnotherRecord && !column.system) {
const colOpt = await column.getColOptions<LinkToAnotherRecordColumn>();
if (colOpt.type !== RelationTypes.BELONGS_TO) {
return [
...(await paramsArr),
nestedWhereParam(column.title),
nestedOffsetParam(column.title),
nestedLimitParam(column.title),
nestedFieldParam(column.title),
nestedSortParam(column.title),
];
} else {
return [...(await paramsArr), nestedFieldParam(column.title)];
}
}
return paramsArr;
}, Promise.resolve([]));
};

407
packages/nocodb/src/services/api-docs/swaggerV2/templates/paths.ts

@ -0,0 +1,407 @@
import { ModelTypes } from 'nocodb-sdk';
import {
fieldsParam,
getNestedParams,
limitParam,
linkFieldNameParam,
offsetParam,
recordIdParam,
shuffleParam,
sortParam,
viewIdParams,
whereParam,
} from './params';
import type { SwaggerColumn } from '../getSwaggerColumnMetas';
import type { SwaggerView } from '~/services/api-docs/swaggerV2/getSwaggerJSONV2';
import { isRelationExist } from '~/services/api-docs/swagger/templates/paths';
export const getModelPaths = async (ctx: {
tableName: string;
type: ModelTypes;
columns: SwaggerColumn[];
tableId: string;
views: SwaggerView[];
}): Promise<{ [path: string]: any }> => ({
[`/api/v2/tables/${ctx.tableId}/records`]: {
get: {
summary: `${ctx.tableName} list`,
operationId: `${ctx.tableName.toLowerCase()}-db-table-row-list`,
description: `List of all rows from ${ctx.tableName} ${ctx.type} and response data fields can be filtered based on query params.`,
tags: [ctx.tableName],
parameters: [
viewIdParams(ctx.views),
fieldsParam,
sortParam,
whereParam,
limitParam,
shuffleParam,
offsetParam,
...(await getNestedParams(ctx.columns)),
],
responses: {
'200': {
description: 'OK',
content: {
'application/json': {
schema: getPaginatedResponseType(`${ctx.tableName}Response`),
},
},
},
},
},
...(ctx.type === ModelTypes.TABLE
? {
post: {
summary: `${ctx.tableName} create`,
description:
'Insert a new row in table by providing a key value pair object where key refers to the column alias. All the required fields should be included with payload excluding `autoincrement` and column with default value.',
operationId: `${ctx.tableName.toLowerCase()}-create`,
responses: {
'200': {
description: 'OK',
content: {
'application/json': {
schema: {
$ref: `#/components/schemas/${ctx.tableName}Response`,
},
},
},
},
'400': {
$ref: '#/components/responses/BadRequest',
},
},
tags: [ctx.tableName],
requestBody: {
content: {
'application/json': {
schema: {
oneOf: [
{
$ref: `#/components/schemas/${ctx.tableName}Request`,
},
{
type: 'array',
items: {
$ref: `#/components/schemas/${ctx.tableName}Request`,
},
},
],
},
},
},
},
},
patch: {
summary: `${ctx.tableName} update`,
operationId: `${ctx.tableName.toLowerCase()}-update`,
description:
'Partial update row in table by providing a key value pair object where key refers to the column alias. You need to only include columns which you want to update.',
responses: {
'200': {
description: 'OK',
content: {
'application/json': {
schema: {},
},
},
},
'400': {
$ref: '#/components/responses/BadRequest',
},
},
tags: [ctx.tableName],
requestBody: {
content: {
'application/json': {
schema: {
oneOf: [
{
$ref: `#/components/schemas/${ctx.tableName}Request`,
},
{
type: 'array',
items: {
$ref: `#/components/schemas/${ctx.tableName}Request`,
},
},
],
},
},
},
},
},
delete: {
summary: `${ctx.tableName} delete`,
operationId: `${ctx.tableName.toLowerCase()}-delete`,
responses: {
'200': {
description: 'OK',
},
},
tags: [ctx.tableName],
description:
'Delete a row by using the **primary key** column value.',
requestBody: {
content: {
'application/json': {
schema: {
oneOf: [
{
$ref: `#/components/schemas/${ctx.tableName}IdRequest`,
},
{
type: 'array',
items: {
$ref: `#/components/schemas/${ctx.tableName}IdRequest`,
},
},
],
},
},
},
},
},
}
: {}),
},
[`/api/v2/tables/${ctx.tableId}/records/{recordId}`]: {
get: {
parameters: [recordIdParam, fieldsParam],
summary: `${ctx.tableName} read`,
description: 'Read a row data by using the **primary key** column value.',
operationId: `${ctx.tableName.toLowerCase()}-read`,
tags: [ctx.tableName],
responses: {
'201': {
description: 'Created',
content: {
'application/json': {
schema: {
$ref: `#/components/schemas/${ctx.tableName}Response`,
},
},
},
},
},
},
},
[`/api/v2/tables/${ctx.tableId}/records/count`]: {
parameters: [viewIdParams(ctx.views)],
get: {
summary: `${ctx.tableName} count`,
operationId: `${ctx.tableName.toLowerCase()}-count`,
description: 'Get rows count of a table by applying optional filters.',
tags: [ctx.tableName],
parameters: [whereParam],
responses: {
'200': {
description: 'OK',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
count: {
type: 'number',
},
},
required: ['list', 'pageInfo'],
},
examples: {
'Example 1': {
value: {
count: 3,
},
},
},
},
},
},
'400': {
$ref: '#/components/responses/BadRequest',
},
},
},
},
...(isRelationExist(ctx.columns)
? {
[`/api/v2/tables/${ctx.tableId}/links/{linkFieldId}/records/{recordId}`]:
{
parameters: [linkFieldNameParam(ctx.columns), recordIdParam],
get: {
summary: 'Link Records list',
operationId: `${ctx.tableName.toLowerCase()}-nested-list`,
description:
'This API endpoint allows you to retrieve list of linked records for a specific `Link field` and `Record ID`. The response is an array of objects containing Primary Key and its corresponding display value.',
tags: [ctx.tableName],
parameters: [
fieldsParam,
sortParam,
whereParam,
limitParam,
offsetParam,
],
responses: {
'200': {
description: 'OK',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
list: {
type: 'array',
description: 'List of data objects',
items: {
type: 'object',
},
},
pageInfo: {
$ref: '#/components/schemas/Paginated',
description: 'Paginated Info',
},
},
required: ['list', 'pageInfo'],
},
},
},
},
'400': {
$ref: '#/components/responses/BadRequest',
},
},
},
post: {
summary: 'Link Records',
operationId: `${ctx.tableName.toLowerCase()}-nested-link`,
responses: {
'200': {
description: 'OK',
content: {
'application/json': {
schema: {},
examples: {
'Example 1': {
value: true,
},
},
},
},
},
'400': {
$ref: '#/components/responses/BadRequest',
},
},
tags: [ctx.tableName],
requestBody: {
content: {
'application/json': {
schema: {
oneOf: [
{
type: 'object',
},
{
type: 'array',
items: {
type: 'object',
},
},
],
},
examples: {
'Example 1': {
value: [
{
Id: 4,
},
{
Id: 5,
},
],
},
},
},
},
},
description:
'This API endpoint allows you to link records to a specific `Link field` and `Record ID`. The request payload is an array of record-ids from the adjacent table for linking purposes. Note that any existing links, if present, will be unaffected during this operation.',
parameters: [recordIdParam],
},
delete: {
summary: 'Unlink Records',
operationId: `${ctx.tableName.toLowerCase()}-nested-unlink`,
responses: {
'200': {
description: 'OK',
content: {
'application/json': {
schema: {},
examples: {
'Example 1': {
value: true,
},
},
},
},
},
'400': {
$ref: '#/components/responses/BadRequest',
},
},
tags: [ctx.tableName],
requestBody: {
content: {
'application/json': {
schema: {
oneOf: [
{
type: 'array',
items: {
type: 'object',
},
},
],
},
examples: {
'Example 1': {
value: [
{
Id: 1,
},
{
Id: 2,
},
],
},
},
},
},
},
description:
'This API endpoint allows you to unlink records from a specific `Link field` and `Record ID`. The request payload is an array of record-ids from the adjacent table for unlinking purposes. Note that, \n- duplicated record-ids will be ignored.\n- non-existent record-ids will be ignored.',
parameters: [recordIdParam],
},
},
}
: {}),
});
function getPaginatedResponseType(type: string) {
return {
type: 'object',
properties: {
list: {
type: 'array',
items: {
$ref: `#/components/schemas/${type}`,
},
},
PageInfo: {
$ref: `#/components/schemas/Paginated`,
},
},
};
}

109
packages/nocodb/src/services/api-docs/swaggerV2/templates/schemas.ts

@ -0,0 +1,109 @@
import { isSystemColumn } from 'nocodb-sdk';
import type { SwaggerColumn } from '../getSwaggerColumnMetas';
export const getModelSchemas = (ctx: {
tableName: string;
orgs: string;
baseName: string;
columns: Array<SwaggerColumn>;
}) => ({
[`${ctx.tableName}Response`]: {
title: `${ctx.tableName} Response`,
type: 'object',
description: '',
'x-internal': false,
properties: {
...(ctx.columns?.reduce(
(colsObj, { title, virtual, column, ...fieldProps }) => ({
...colsObj,
...(column.system
? {}
: {
[title]: fieldProps,
}),
}),
{},
) || {}),
},
},
[`${ctx.tableName}Request`]: {
title: `${ctx.tableName} Request`,
type: 'object',
description: '',
'x-internal': false,
properties: {
...(ctx.columns?.reduce(
(colsObj, { title, virtual, column, ...fieldProps }) => ({
...colsObj,
...(virtual || isSystemColumn(column) || column.ai || column.meta?.ag
? {}
: {
[title]: fieldProps,
}),
}),
{},
) || {}),
},
},
[`${ctx.tableName}IdRequest`]: {
title: `${ctx.tableName} Id Request`,
type: 'object',
description: '',
'x-internal': false,
properties: {
...(ctx.columns?.reduce(
(colsObj, { title, virtual, column, ...fieldProps }) => ({
...colsObj,
...(column.pk
? {
[title]: fieldProps,
}
: {}),
}),
{},
) || {}),
},
},
});
export const getViewSchemas = (ctx: {
tableName: string;
viewName: string;
orgs: string;
baseName: string;
columns: Array<SwaggerColumn>;
}) => ({
[`${ctx.tableName}${ctx.viewName}GridResponse`]: {
title: `${ctx.tableName} : ${ctx.viewName} Response`,
type: 'object',
description: '',
'x-internal': false,
properties: {
...(ctx.columns?.reduce(
(colsObj, { title, virtual, column, ...fieldProps }) => ({
...colsObj,
[title]: fieldProps,
}),
{},
) || {}),
},
},
[`${ctx.tableName}${ctx.viewName}GridRequest`]: {
title: `${ctx.tableName} : ${ctx.viewName} Request`,
type: 'object',
description: '',
'x-internal': false,
properties: {
...(ctx.columns?.reduce(
(colsObj, { title, virtual, column, ...fieldProps }) => ({
...colsObj,
...(virtual
? {}
: {
[title]: fieldProps,
}),
}),
{},
) || {}),
},
},
});

3
packages/nocodb/src/services/datas.service.ts

@ -27,6 +27,7 @@ export class DatasService {
model,
view,
query: param.query,
throwErrorIfInvalidParams: true,
});
}
@ -51,7 +52,7 @@ export class DatasService {
dbDriver: await NcConnectionMgrv2.get(source),
});
const countArgs: any = { ...param.query };
const countArgs: any = { ...param.query, throwErrorIfInvalidParams: true };
try {
countArgs.filterArr = JSON.parse(countArgs.filterArrJson);
} catch (e) {}

4
packages/nocodb/src/services/tables.service.ts

@ -461,9 +461,7 @@ export class TablesService {
}
if (column.column_name.length > mxColumnLength) {
NcError.badRequest(
`Column name ${column.column_name} exceeds ${mxColumnLength} characters`,
);
column.column_name = column.column_name.slice(0, mxColumnLength);
}
if (column.title && column.title.length > 255) {

5
packages/nocodb/src/utils/globals.ts

@ -171,6 +171,11 @@ export enum CacheDelDirection {
CHILD_TO_PARENT = 'CHILD_TO_PARENT',
}
export const GROUPBY_COMPARISON_OPS = <const>[
// these are used for groupby
'gb_eq',
'gb_null',
];
export const COMPARISON_OPS = <const>[
'eq',
'neq',

10
packages/nocodb/tests/unit/rest/tests/groupby.test.ts

@ -299,7 +299,7 @@ function groupByTests() {
expect(response.body.list.length).to.equal(1);
});
it('Check One GroupBy Column with MM Lookup which is not supported', async function () {
it('Check One GroupBy Column with MM Lookup which is supported', async function () {
await createLookupColumn(context, {
base: sakilaProject,
title: 'ActorNames',
@ -308,15 +308,17 @@ function groupByTests() {
relatedTableColumnTitle: 'FirstName',
});
const res = await request(context.app)
const response = await request(context.app)
.get(`/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/groupby`)
.set('xc-auth', context.token)
.query({
column_name: 'ActorNames',
})
.expect(400);
.expect(200);
assert.match(res.body.msg, /not supported/);
assert.match(response.body.list[1]['ActorNames'], /ADAM|ANNE/);
expect(+response.body.list[1]['count']).to.gt(0);
expect(response.body.list.length).to.equal(25);
});
it('Check One GroupBy Column with Formula and Formula referring another formula', async function () {

491
pnpm-lock.yaml

File diff suppressed because it is too large Load Diff

3
renovate.json

@ -3,7 +3,6 @@
"extends": [
"config:base",
":dependencyDashboard",
":onlyNpm",
":prConcurrentLimit20",
":autodetectPinVersions",
":label(renovate)",
@ -62,5 +61,5 @@
"assignees": [
"wingkwong"
],
"enabled": false
"enabled": true
}

6
tests/playwright/pages/Dashboard/Import/ImportTemplate.ts

@ -23,7 +23,11 @@ export class ImportTemplatePage extends BasePage {
const rowCount = await tr.count();
const tableList: string[] = [];
for (let i = 0; i < rowCount; i++) {
const tableName = await getTextExcludeIconText(tr.nth(i));
const tableName = await this.get()
.locator(`.ant-collapse-header`)
.nth(i)
.locator('input[type="text"]')
.inputValue();
tableList.push(tableName);
}
return tableList;

Loading…
Cancel
Save