From aa47e4a176ee184f23c0ee29c44b70e0ec8e48ed Mon Sep 17 00:00:00 2001 From: Mert E Date: Sun, 30 Jun 2024 13:11:45 +0300 Subject: [PATCH 01/51] fix: long base_id issue (#8884) * fix: long base_id issue * refactor: variable names --- .../meta/migrations/XcMigrationSourcev2.ts | 4 + .../migrations/v2/nc_050_tenant_isolation.ts | 4 + .../meta/migrations/v2/nc_054_id_length.ts | 200 ++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 packages/nocodb/src/meta/migrations/v2/nc_054_id_length.ts diff --git a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts index b6f946ced4..8606cda5c0 100644 --- a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts +++ b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts @@ -40,6 +40,7 @@ import * as nc_050_tenant_isolation from '~/meta/migrations/v2/nc_050_tenant_iso import * as nc_051_source_readonly_columns from '~/meta/migrations/v2/nc_051_source_readonly_columns'; import * as nc_052_field_aggregation from '~/meta/migrations/v2/nc_052_field_aggregation'; import * as nc_053_jobs from '~/meta/migrations/v2/nc_053_jobs'; +import * as nc_054_id_length from '~/meta/migrations/v2/nc_054_id_length'; // Create a custom migration source class export default class XcMigrationSourcev2 { @@ -91,6 +92,7 @@ export default class XcMigrationSourcev2 { 'nc_051_source_readonly_columns', 'nc_052_field_aggregation', 'nc_053_jobs', + 'nc_054_id_length', ]); } @@ -184,6 +186,8 @@ export default class XcMigrationSourcev2 { return nc_052_field_aggregation; case 'nc_053_jobs': return nc_053_jobs; + case 'nc_054_id_length': + return nc_054_id_length; } } } diff --git a/packages/nocodb/src/meta/migrations/v2/nc_050_tenant_isolation.ts b/packages/nocodb/src/meta/migrations/v2/nc_050_tenant_isolation.ts index 270bfe174e..7473429145 100644 --- a/packages/nocodb/src/meta/migrations/v2/nc_050_tenant_isolation.ts +++ b/packages/nocodb/src/meta/migrations/v2/nc_050_tenant_isolation.ts @@ -1,5 +1,6 @@ import type { Knex } from 'knex'; import { MetaTable } from '~/utils/globals'; +import { replaceLongBaseIds } from '~/meta/migrations/v2/nc_054_id_length'; /* Add base_id to: @@ -181,6 +182,9 @@ const listIndexesOnColumn = async ( const up = async (knex: Knex) => { log('Migration started'); + // Replace long base_ids before adding new columns to avoid value too long error + await replaceLongBaseIds(knex); + log('Adding missing base_id columns'); const addBaseId = [ diff --git a/packages/nocodb/src/meta/migrations/v2/nc_054_id_length.ts b/packages/nocodb/src/meta/migrations/v2/nc_054_id_length.ts new file mode 100644 index 0000000000..3ac502cc84 --- /dev/null +++ b/packages/nocodb/src/meta/migrations/v2/nc_054_id_length.ts @@ -0,0 +1,200 @@ +import { customAlphabet } from 'nanoid'; +import type { Knex } from 'knex'; +import { MetaTable } from '~/utils/globals'; + +const log = (message: string) => { + console.log(`nc_054_id_length: ${message}`); +}; + +let hrTime = process.hrtime(); + +const logExecutionTime = (message: string) => { + const [seconds, nanoseconds] = process.hrtime(hrTime); + const elapsedSeconds = seconds + nanoseconds / 1e9; + log(`${message} in ${elapsedSeconds}s`); +}; + +const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14); + +const generateUniqueBaseId = async (knex: Knex) => { + const baseId = `p${nanoidv2()}`; + + const base = await knex(MetaTable.PROJECT).where('id', baseId).first(); + + if (base) { + return generateUniqueBaseId(knex); + } + + return baseId; +}; + +const listBasesWithLongIds = async (knex: Knex) => { + const sourceType = knex.client.driverName; + + switch (sourceType) { + case 'pg': { + const bases = await knex.raw( + `SELECT id FROM ?? WHERE LENGTH(id) > 20`, + MetaTable.PROJECT, + ); + + return bases.rows.map((row: any) => row.id); + } + case 'mysql': + case 'mysql2': { + const bases = await knex.raw( + `SELECT id FROM ?? WHERE CHAR_LENGTH(id) > 20`, + MetaTable.PROJECT, + ); + + return bases[0].map((row: any) => row.id); + } + case 'sqlite3': { + const bases = await knex.raw( + `SELECT id FROM ?? WHERE LENGTH(id) > 20`, + MetaTable.PROJECT, + ); + + return bases.map((row: any) => row.id); + } + case 'mssql': { + const bases = await knex.raw( + `SELECT id FROM ?? WHERE LEN(id) > 20`, + MetaTable.PROJECT, + ); + + return bases.map((row: any) => row.id); + } + + default: + throw new Error(`Unsupported database: ${sourceType}`); + } +}; + +export const replaceLongBaseIds = async (knex: Knex) => { + const basesWithLongIds = await listBasesWithLongIds(knex); + + for (const baseId of basesWithLongIds) { + const newBaseId = await generateUniqueBaseId(knex); + + if (!baseId || !newBaseId) { + throw new Error(`Failed to replace ${baseId} with ${newBaseId}`); + } + + const tablesToChangeBaseId = [ + MetaTable.API_TOKENS, + MetaTable.AUDIT, + MetaTable.PROJECT_USERS, + MetaTable.CALENDAR_VIEW_COLUMNS, + MetaTable.CALENDAR_VIEW, + MetaTable.COLUMNS, + MetaTable.COMMENTS_REACTIONS, + MetaTable.COMMENTS, + MetaTable.MODEL_ROLE_VISIBILITY, + MetaTable.EXTENSIONS, + MetaTable.FILTER_EXP, + MetaTable.FORM_VIEW_COLUMNS, + MetaTable.FORM_VIEW, + MetaTable.GALLERY_VIEW_COLUMNS, + MetaTable.GALLERY_VIEW, + MetaTable.GRID_VIEW_COLUMNS, + MetaTable.GRID_VIEW, + MetaTable.HOOK_LOGS, + MetaTable.HOOKS, + MetaTable.KANBAN_VIEW_COLUMNS, + MetaTable.KANBAN_VIEW, + MetaTable.MAP_VIEW_COLUMNS, + MetaTable.MAP_VIEW, + MetaTable.MODELS, + MetaTable.SORT, + MetaTable.BASES, + MetaTable.SYNC_LOGS, + MetaTable.SYNC_SOURCE, + MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE, + MetaTable.VIEWS, + ]; + + for (const table of tablesToChangeBaseId) { + await knex(table).where('base_id', baseId).update({ base_id: newBaseId }); + } + + await knex(MetaTable.PROJECT).where('id', baseId).update({ id: newBaseId }); + + log(`Replaced ${baseId} with ${newBaseId} (because it was too long)`); + } +}; + +const tablesToAlterBaseId = [ + MetaTable.API_TOKENS, + MetaTable.AUDIT, + MetaTable.PROJECT_USERS, + MetaTable.COLUMNS, + MetaTable.COMMENTS_REACTIONS, + MetaTable.COMMENTS, + MetaTable.MODEL_ROLE_VISIBILITY, + MetaTable.FILTER_EXP, + MetaTable.FORM_VIEW_COLUMNS, + MetaTable.FORM_VIEW, + MetaTable.GALLERY_VIEW_COLUMNS, + MetaTable.GALLERY_VIEW, + MetaTable.GRID_VIEW_COLUMNS, + MetaTable.GRID_VIEW, + MetaTable.HOOK_LOGS, + MetaTable.HOOKS, + MetaTable.KANBAN_VIEW_COLUMNS, + MetaTable.KANBAN_VIEW, + MetaTable.MAP_VIEW, + MetaTable.MAP_VIEW_COLUMNS, + MetaTable.MODELS, + MetaTable.SORT, + MetaTable.BASES, + MetaTable.SYNC_LOGS, + MetaTable.SYNC_SOURCE, + MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE, + MetaTable.VIEWS, +]; + +const tablesToAlterSourceId = [ + MetaTable.CALENDAR_VIEW, + MetaTable.CALENDAR_VIEW_COLUMNS, +]; + +const up = async (knex: Knex) => { + hrTime = process.hrtime(); + + await replaceLongBaseIds(knex); + + logExecutionTime('Replaced long base IDs'); + + for (const table of tablesToAlterBaseId) { + hrTime = process.hrtime(); + await knex.schema.alterTable(table, (tableQb) => { + tableQb.string('base_id', 20).alter(); + }); + logExecutionTime(`Altered ${table}.base_id to 20 characters`); + } + + for (const table of tablesToAlterSourceId) { + hrTime = process.hrtime(); + await knex.schema.alterTable(table, (tableQb) => { + tableQb.string('source_id', 20).alter(); + }); + logExecutionTime(`Altered ${table}.source_id to 20 characters`); + } +}; + +const down = async (knex: Knex) => { + for (const table of tablesToAlterBaseId) { + await knex.schema.alterTable(table, (tableQb) => { + tableQb.string('base_id', 128).alter(); + }); + } + + for (const table of tablesToAlterSourceId) { + await knex.schema.alterTable(table, (tableQb) => { + tableQb.string('source_id', 128).alter(); + }); + } +}; + +export { up, down }; From bd00e8e24344387bdf3349c68a1355fcdb50afe5 Mon Sep 17 00:00:00 2001 From: Anbarasu Date: Mon, 1 Jul 2024 17:02:41 +0530 Subject: [PATCH 02/51] feat: groupby aggregation (#8838) * feat: groupby aggregation * fix: reload data * fix(nc-gui): fix: better aggregate update condition * fix(nc-gui): limit api aggregation calls * fix(nocodb): aggregation api refactor feat(nocodb): bulk aggregate apis * feat(nocodb): update swagger.json fix(nocodb): parse aggregation json feat(nocodb): public endpoint * fix: get first res * fix: swagger bugs * fix: integrate bulk aggregation * fix(nc-gui): bulk agg not triggering in shared view * fix(nc-gui): avoid agg load inside table in groupby * fix(nc-gui): issue when key is decimal * fix(nc-gui): trigger aggregate before ui update * fix(nc-gui): update triggering bulk api calls * fix(nc-gui): custom label in pagination * test: unit tests * chore: move to ee * fix: gui separation * fix: pagination gui * fix: summary flicker * fix: pr review * fix: groupby pagination * fix: aggregation alias handling * fix: agg not updating * fix: expanded-form broken * fix: unit tests * fix: playwright tests * fix: playwright tests * fix: user select, groupby tests * fix: groupby inner pagination alignment * fix: stddev sqlite value * fix: review changes * fix: ui flicker --- .../nc-gui/components/nc/PaginationV2.vue | 2 +- .../nc-gui/components/shared-view/Grid.vue | 2 + .../components/smartsheet/Pagination.vue | 8 +- .../smartsheet/grid/Aggregation.vue | 13 + .../components/smartsheet/grid/GroupBy.vue | 104 +++-- .../smartsheet/grid/GroupByTable.vue | 183 ++++++++- .../smartsheet/grid/PaginationV2.vue | 39 +- .../components/smartsheet/grid/Table.vue | 7 +- .../components/smartsheet/grid/index.vue | 21 +- .../smartsheet/toolbar/GroupByMenu.vue | 30 +- packages/nc-gui/composables/useData.ts | 4 +- packages/nc-gui/composables/useSharedView.ts | 30 ++ .../nc-gui/composables/useViewAggregate.ts | 26 +- packages/nc-gui/composables/useViewGroupBy.ts | 91 ++++- packages/nc-gui/context/index.ts | 14 +- packages/nc-gui/lang/en.json | 2 +- packages/nc-gui/lib/types.ts | 1 + .../src/controllers/data-table.controller.ts | 2 +- .../controllers/public-datas.controller.ts | 2 +- packages/nocodb/src/db/BaseModelSqlv2.ts | 185 ++++++++- packages/nocodb/src/db/aggregation.ts | 5 + packages/nocodb/src/db/aggregations/mysql2.ts | 143 ++++--- packages/nocodb/src/db/aggregations/pg.ts | 145 +++---- .../nocodb/src/db/aggregations/sqlite3.ts | 139 +++---- packages/nocodb/src/schema/swagger-v2.json | 102 +++++ .../nocodb/src/services/data-table.service.ts | 4 +- packages/nocodb/tests/unit/factory/column.ts | 154 +++++++ packages/nocodb/tests/unit/rest/index.test.ts | 5 + .../tests/unit/rest/tests/aggregation.test.ts | 384 ++++++++++++++++++ .../playwright/pages/Dashboard/Grid/Group.ts | 8 +- .../pages/Dashboard/common/Toolbar/Groupby.ts | 2 +- .../tests/db/columns/columnUserSelect.spec.ts | 3 +- .../db/general/toolbarOperations.spec.ts | 11 +- 33 files changed, 1561 insertions(+), 310 deletions(-) create mode 100644 packages/nc-gui/components/smartsheet/grid/Aggregation.vue create mode 100644 packages/nocodb/tests/unit/rest/tests/aggregation.test.ts diff --git a/packages/nc-gui/components/nc/PaginationV2.vue b/packages/nc-gui/components/nc/PaginationV2.vue index f6ba603cfc..da04355732 100644 --- a/packages/nc-gui/components/nc/PaginationV2.vue +++ b/packages/nc-gui/components/nc/PaginationV2.vue @@ -145,7 +145,7 @@ const pageSizeOptions = [ + From 5cfe0f781bc9ea547d218cbfb66ff62a1d35c66e Mon Sep 17 00:00:00 2001 From: Ramesh Mane <101566080+rameshmane7218@users.noreply.github.com> Date: Sat, 6 Jul 2024 07:08:44 +0530 Subject: [PATCH 50/51] Nc feature: Import CSV - skip field mapping during import (#8915) * fix(nc-gui): import csv changes * fix(nc-gui): tooltip alignment issue * fix(nc-gui): duplicate table name issue * fix(nc-gui): uncheck auto select field type checkbox * fix(nc-gui): skip field mapping during csv import * chore(nc-gui): lint * fix(nc-gui): small changes * fix(nc-gui): remove auto mapping option * fix(test): update import excel test cases * fix(nc-gui): review changes * fix(nc-gui): quick import modal width issue * fix(nc-gui): w.replace is not a function --- .../nc-gui/components/dlg/QuickImport.vue | 58 +++-- .../nc-gui/components/template/Editor.vue | 215 ++++++++---------- packages/nc-gui/components/template/utils.ts | 11 +- .../helpers/parsers/ExcelTemplateAdapter.ts | 10 +- packages/nc-gui/lang/en.json | 5 +- .../tests/db/features/import.spec.ts | 5 +- 6 files changed, 142 insertions(+), 162 deletions(-) diff --git a/packages/nc-gui/components/dlg/QuickImport.vue b/packages/nc-gui/components/dlg/QuickImport.vue index c5d045e182..b1cc957012 100644 --- a/packages/nc-gui/components/dlg/QuickImport.vue +++ b/packages/nc-gui/components/dlg/QuickImport.vue @@ -70,6 +70,7 @@ const defaultImportState = { autoSelectFieldTypes: true, firstRowAsHeaders: true, shouldImportData: true, + importDataOnly: true, }, } const importState = reactive(defaultImportState) @@ -84,7 +85,6 @@ const IsImportTypeExcel = computed(() => importType === 'excel') const validators = computed(() => ({ url: [fieldRequiredValidator(), importUrlValidator, isImportTypeCsv.value ? importCsvUrlValidator : importExcelUrlValidator], - maxRowsToParse: [fieldRequiredValidator()], })) const { validate, validateInfos } = useForm(importState, validators) @@ -152,10 +152,6 @@ const disableImportButton = computed(() => !templateEditorRef.value?.isValid || const disableFormatJsonButton = computed(() => !jsonEditorRef.value?.isValid) const modalWidth = computed(() => { - if (importType === 'excel' && templateEditorModal.value) { - return 'max(90vw, 600px)' - } - return 'max(60vw, 600px)' }) @@ -254,9 +250,10 @@ function formatJson() { jsonEditorRef.value?.format() } -function populateUniqueTableName(tn: string) { +function populateUniqueTableName(tn: string, draftTn: string[] = []) { let c = 1 while ( + draftTn.includes(tn) || baseTables.value.get(baseId)?.some((t: TableType) => { const s = t.table_name.split('___') let target = t.table_name @@ -492,10 +489,13 @@ async function parseAndExtractData(val: UploadFile[] | ArrayBuffer | string) { if (importDataOnly) importColumns.value = templateGenerator!.getColumns() else { // ensure the target table name not exist in current table list - templateData.value.tables = templateData.value.tables.map((table: Record) => ({ - ...table, - table_name: populateUniqueTableName(table.table_name), - })) + const draftTableNames = [] as string[] + + templateData.value.tables = templateData.value.tables.map((table: Record) => { + const table_name = populateUniqueTableName(table.table_name, draftTableNames) + draftTableNames.push(table_name) + return { ...table, table_name } + }) } importData.value = templateGenerator!.getData() } @@ -517,6 +517,11 @@ const onError = () => { const onChange = () => { isError.value = false } + +onMounted(() => { + importState.parserConfig.importDataOnly = importDataOnly + importState.parserConfig.autoSelectFieldTypes = importDataOnly +})
- + - +
@@ -625,16 +638,6 @@ const onChange = () => { {{ $t('title.advancedSettings') }} - - - - - - - {{ $t('labels.autoSelectFieldTypes') }} - - - {{ $t('labels.firstRowAsHeaders') }} @@ -699,3 +702,12 @@ const onChange = () => { + + diff --git a/packages/nc-gui/components/template/Editor.vue b/packages/nc-gui/components/template/Editor.vue index ff5842f36d..052b96ff37 100644 --- a/packages/nc-gui/components/template/Editor.vue +++ b/packages/nc-gui/components/template/Editor.vue @@ -101,7 +101,7 @@ const isImporting = ref(false) const importingTips = ref>({}) -const checkAllRecord = ref([]) +const checkAllRecord = ref>({}) const formError = ref() @@ -168,14 +168,20 @@ watch( let res = true if (importDataOnly) { for (const tn of Object.keys(srcDestMapping.value)) { + let flag = false if (!atLeastOneEnabledValidation(tn)) { res = false } for (const record of srcDestMapping.value[tn]) { if (!fieldsValidation(record, tn)) { - return false + res = false + flag = true + break } } + if (flag) { + break + } } } else { for (const [_, o] of Object.entries(validateInfos)) { @@ -258,20 +264,13 @@ function deleteTable(tableIdx: number) { function deleteTableColumn(tableIdx: number, columnKey: number) { const columnIdx = data.tables[tableIdx].columns.findIndex((c: ColumnType & { key: number }) => c.key === columnKey) data.tables[tableIdx].columns.splice(columnIdx, 1) -} - -function addNewColumnRow(tableIdx: number, uidt: string) { - data.tables[tableIdx].columns.push({ - key: data.tables[tableIdx].columns.length, - title: `title${data.tables[tableIdx].columns.length + 1}`, - column_name: `title${data.tables[tableIdx].columns.length + 1}`, - uidt, - }) + let key = 0 - nextTick(() => { - const input = inputRefs.value[data.tables[tableIdx].columns.length - 1] - input.focus() - input.select() + data.tables[tableIdx].columns.forEach((_c: ColumnType & { key: number }, i: number) => { + if (data.tables[tableIdx].columns[i].key !== undefined) { + data.tables[tableIdx].columns[i].key = key + key++ + } }) } @@ -763,7 +762,7 @@ watch(modelRef, async () => { {