From ea41065366abc744efe151080f8bf00f9d9b9d67 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Wed, 18 May 2022 11:53:01 +0530 Subject: [PATCH] Feat/at sync test: migration fixes (#2068) * fix: ignore rollup for lookup columns Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * test/cypress: base file for import airtable verification Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * test/cypress: import access creds Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * fix: multiselect as csv string instead of array Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * fix: ludicrous mode view data Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * fix: use decimal if precision configured, support for default form view Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * fix: handle ? in column name Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * refactor: migration logs Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../components/import/ImportFromAirtable.vue | 9 ++- .../lib/dataMapper/lib/sql/BaseModelSqlv2.ts | 2 +- .../src/lib/noco/meta/api/sync/helpers/job.ts | 73 ++++++++++++------- scripts/cypress/cypress.json | 6 +- .../common/7b_import_from_airtable.js | 62 ++++++++++++++++ 5 files changed, 121 insertions(+), 31 deletions(-) create mode 100644 scripts/cypress/integration/common/7b_import_from_airtable.js diff --git a/packages/nc-gui/components/import/ImportFromAirtable.vue b/packages/nc-gui/components/import/ImportFromAirtable.vue index 00003e9cea..c204f036d2 100644 --- a/packages/nc-gui/components/import/ImportFromAirtable.vue +++ b/packages/nc-gui/components/import/ImportFromAirtable.vue @@ -5,7 +5,7 @@

{{ $t('title.importFromAirtable') }}

-
+
🚀
@@ -30,7 +30,7 @@ outlined dense label="Api Key" - class="caption" + class="caption nc-input-api-key" :type="isPasswordVisible ? 'text':'password'" :rules="[v=> !!v || 'Api Key is required']" > @@ -45,13 +45,14 @@ outlined dense label="Shared Base ID / URL" - class="caption" + class="caption nc-input-shared-base" :rules="[(v) => !!v || 'Shared Base ID / URL is required']" />
- + Go to dashboard diff --git a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts index e9f781fb10..ecfd717524 100644 --- a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts @@ -1188,7 +1188,7 @@ class BaseModelSqlv2 { break; default: res[ - column.title || column.column_name + (column.title || column.column_name).replace(/\?/g, '\\?') ] = `${this.model.table_name}.${column.column_name}`; break; } diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index 99557902ad..484f979ff0 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -73,15 +73,15 @@ export default async ( count: 0, time: 0 }, - columnNotMigrated: { + migrationSkipLog: { count: 0, log: [] } }; - function addToSkipColumnLog(tbl, col, type, reason?) { - runTimeCounters.columnNotMigrated.count++; - runTimeCounters.columnNotMigrated.log.push( + function updateMigrationSkipLog(tbl, col, type, reason?) { + runTimeCounters.migrationSkipLog.count++; + runTimeCounters.migrationSkipLog.log.push( `tn[${tbl}] cn[${col}] type[${type}] :: ${reason}` ); } @@ -305,6 +305,7 @@ export default async ( ncType = UITypes.Duration; else if (col.typeOptions?.format === 'currency') ncType = UITypes.Currency; + else if (col.typeOptions?.precision > 0) ncType = UITypes.Decimal; break; case 'formula': @@ -411,7 +412,7 @@ export default async ( // not supported datatype: pure formula field // allow formula based computed fields (created time/ modified time to go through) if (ncCol.uidt === UITypes.Formula) { - addToSkipColumnLog( + updateMigrationSkipLog( tblSchema[i].name, ncName.title, col.type, @@ -736,7 +737,7 @@ export default async ( if (enableErrorLogs) console.log(`## Invalid column IDs mapped; skip`); - addToSkipColumnLog( + updateMigrationSkipLog( srcTableSchema.title, aTblColumns[i].name, aTblColumns[i].type, @@ -794,7 +795,7 @@ export default async ( const fTblField = nestedLookupTbl[i].typeOptions.foreignTableRollupColumnId; const name = aTbl_getColumnName(fTblField); - addToSkipColumnLog( + updateMigrationSkipLog( ncSchema.tablesById[nestedLookupTbl[i].srcTableId]?.title, nestedLookupTbl[i].name, nestedLookupTbl[i].type, @@ -908,7 +909,7 @@ export default async ( ); if (ncRollupFn === '') { - addToSkipColumnLog( + updateMigrationSkipLog( srcTableSchema.title, aTblColumns[i].name, aTblColumns[i].type, @@ -924,7 +925,7 @@ export default async ( if (enableErrorLogs) console.log(`## Invalid column IDs mapped; skip`); - addToSkipColumnLog( + updateMigrationSkipLog( srcTableSchema.title, aTblColumns[i].name, aTblColumns[i].type, @@ -946,6 +947,24 @@ export default async ( continue; } + // skip, if rollup column was pointing to another virtual column + const ncColSchema = await nc_getColumnSchema( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + if ( + ncColSchema?.uidt === UITypes.Formula || + ncColSchema?.uidt === UITypes.Lookup || + ncColSchema?.uidt === UITypes.Rollup + ) { + updateMigrationSkipLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + 'rollup referring to a lookup column' + ); + continue; + } + const ncName = nc_getSanitizedColumnName( srcTableSchema, aTblColumns[i].name @@ -1190,6 +1209,8 @@ export default async ( rec[key] = atDateField.utc().format('YYYY-MM-DD HH:mm'); } + if (dt === UITypes.MultiSelect) rec[key] = value.join(','); + if (dt === UITypes.Attachment) { const tempArr = []; for (const v of value) { @@ -1424,11 +1445,12 @@ export default async ( // response will not include form object if everything is default // if (vData.metadata?.form) { - refreshMode = vData.metadata.form.refreshAfterSubmit; - msg = vData.metadata.form?.afterSubmitMessage - ? vData.metadata.form.afterSubmitMessage - : 'Thank you for submitting the form!'; - desc = vData.metadata.form.description; + if (vData.metadata.form?.refreshAfterSubmit) + refreshMode = vData.metadata.form.refreshAfterSubmit; + if (vData.metadata.form?.afterSubmitMessage) + msg = vData.metadata.form.afterSubmitMessage; + if (vData.metadata.form?.description) + desc = vData.metadata.form.description; } const formData = { @@ -1490,7 +1512,7 @@ export default async ( x => x.id === gridViews[i].id )?.name; const viewList: any = await api.dbView.list(tblId); - const ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id; + let ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id; // create view (default already created) if (i > 0) { @@ -1506,6 +1528,7 @@ export default async ( tblId ); // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Create ${viewName}`) + ncViewId = viewCreated.id; } // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Hide columns ${viewName}`) @@ -1724,8 +1747,11 @@ export default async ( // column not available; // one of not migrated column; if (!colSchema) { - addUserInfo( - `Filter configuration partial: ${sMap.getNcNameFromAtId(viewId)}` + updateMigrationSkipLog( + sMap.getNcNameFromAtId(viewId), + colSchema.title, + colSchema.uidt, + `filter config skipped; column not migrated` ); continue; } @@ -1737,8 +1763,11 @@ export default async ( // console.log(filter) if (datatype === UITypes.Date) { // skip filters over data datatype - addUserInfo( - `Filter configuration partial: ${sMap.getNcNameFromAtId(viewId)}` + updateMigrationSkipLog( + sMap.getNcNameFromAtId(viewId), + colSchema.title, + colSchema.uidt, + `filter config skipped; filter over date datatype not supported` ); continue; } @@ -1861,12 +1890,6 @@ export default async ( } /////////////////////////////////////////////////////////////////////////////// - const userInfo = []; - - function addUserInfo(log) { - userInfo.push(log); - } - try { syncLog = progress; progress('SDK initialized'); diff --git a/scripts/cypress/cypress.json b/scripts/cypress/cypress.json index f1e83f0797..745f35168f 100644 --- a/scripts/cypress/cypress.json +++ b/scripts/cypress/cypress.json @@ -43,7 +43,11 @@ "user": "root", "password": "password" }, - "screenshot": false + "screenshot": false, + "airtable": { + "apiKey": "keyn1MR87qgyUsYg4", + "sharedBase": "https://airtable.com/shrkSQdtKNzUfAbIY" + } }, "fixturesFolder": "scripts/cypress/fixtures", "integrationFolder": "scripts/cypress/integration", diff --git a/scripts/cypress/integration/common/7b_import_from_airtable.js b/scripts/cypress/integration/common/7b_import_from_airtable.js new file mode 100644 index 0000000000..76c66849a1 --- /dev/null +++ b/scripts/cypress/integration/common/7b_import_from_airtable.js @@ -0,0 +1,62 @@ +// Cypress test suite: Project import from Airtable +// + +import { isTestSuiteActive } from "../../support/page_objects/projectConstants"; +let apiKey = "" +let sharedBase = "" + +export const genTest = (apiType, dbType) => { + if (!isTestSuiteActive(apiType, dbType)) return; + + describe(`Import from airtable`, () => { + before(() => { + apiKey = Cypress.env("airtable").apiKey; + sharedBase = Cypress.env("airtable").sharedBase; + }); + + after(() => {}); + + it("Import", () => { + cy.log(apiKey, sharedBase); + + // trigger import + cy.get(".add-btn").click(); + cy.getActiveMenu().contains("Airtable").should('exist').click(); + + // enable turbo + // cy.getActiveModal().find(".nc-btn-enable-turbo").should('exist').click() + + cy.getActiveModal().find(".nc-input-api-key").should('exist').clear().type(apiKey) + cy.getActiveModal().find(".nc-input-shared-base").should('exist').clear().type(sharedBase) + cy.getActiveModal().find(".nc-btn-airtable-import").should('exist').click() + + // it will take a while for import to finish + cy.getActiveModal().find(".nc-btn-go-dashboard", {timeout: 120000}).should('exist').click() + + }); + it("Verify Schema", () => {}); + it("Verify Data", () => {}); + }); +}; + +/** + * @copyright Copyright (c) 2021, Xgene Cloud Ltd + * + * @author Raju Udava + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */