Browse Source

Feat/at sync test: migration fixes (#2068)

* fix: ignore rollup for lookup columns

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* test/cypress: base file for import airtable verification

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* test/cypress: import access creds

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: multiselect as csv string instead of array

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: ludicrous mode view data

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: use decimal if precision configured, support for default form view

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: handle ? in column name

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* refactor: migration logs

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>
pull/2071/head
Raju Udava 2 years ago committed by GitHub
parent
commit
ea41065366
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 9
      packages/nc-gui/components/import/ImportFromAirtable.vue
  2. 2
      packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts
  3. 73
      packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts
  4. 6
      scripts/cypress/cypress.json
  5. 62
      scripts/cypress/integration/common/7b_import_from_airtable.js

9
packages/nc-gui/components/import/ImportFromAirtable.vue

@ -5,7 +5,7 @@
<h3 class="mt-2">
{{ $t('title.importFromAirtable') }}
</h3>
<div v-t="['c:airtable-import:turbo-mode']" class="ml-2 mt-3 title pointer" @click="enableTurbo">
<div v-t="['c:airtable-import:turbo-mode']" class="ml-2 mt-3 title pointer nc-btn-enable-turbo" @click="enableTurbo">
🚀
</div>
<v-spacer />
@ -30,7 +30,7 @@
outlined
dense
label="Api Key"
class="caption"
class="caption nc-input-api-key"
:type="isPasswordVisible ? 'text':'password'"
:rules="[v=> !!v || 'Api Key is required']"
>
@ -45,13 +45,14 @@
outlined
dense
label="Shared Base ID / URL"
class="caption"
class="caption nc-input-shared-base"
:rules="[(v) => !!v || 'Shared Base ID / URL is required']"
/>
</div>
</v-form> <v-card-actions class="justify-center pb-6">
<v-btn
v-t="['c:sync-airtable:save-and-sync']"
class="nc-btn-airtable-import"
:disabled="!valid"
large
color="primary"
@ -103,7 +104,7 @@
v-if="progress && progress.length && progress[progress.length-1].status === 'COMPLETED'"
class="pa-4 pt-8 text-center"
>
<v-btn large color="primary" @click="airtableModal=false">
<v-btn large color="primary" class="nc-btn-go-dashboard" @click="airtableModal=false">
Go to dashboard
</v-btn>
</div>

2
packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts

@ -1188,7 +1188,7 @@ class BaseModelSqlv2 {
break;
default:
res[
column.title || column.column_name
(column.title || column.column_name).replace(/\?/g, '\\?')
] = `${this.model.table_name}.${column.column_name}`;
break;
}

73
packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts

@ -73,15 +73,15 @@ export default async (
count: 0,
time: 0
},
columnNotMigrated: {
migrationSkipLog: {
count: 0,
log: []
}
};
function addToSkipColumnLog(tbl, col, type, reason?) {
runTimeCounters.columnNotMigrated.count++;
runTimeCounters.columnNotMigrated.log.push(
function updateMigrationSkipLog(tbl, col, type, reason?) {
runTimeCounters.migrationSkipLog.count++;
runTimeCounters.migrationSkipLog.log.push(
`tn[${tbl}] cn[${col}] type[${type}] :: ${reason}`
);
}
@ -305,6 +305,7 @@ export default async (
ncType = UITypes.Duration;
else if (col.typeOptions?.format === 'currency')
ncType = UITypes.Currency;
else if (col.typeOptions?.precision > 0) ncType = UITypes.Decimal;
break;
case 'formula':
@ -411,7 +412,7 @@ export default async (
// not supported datatype: pure formula field
// allow formula based computed fields (created time/ modified time to go through)
if (ncCol.uidt === UITypes.Formula) {
addToSkipColumnLog(
updateMigrationSkipLog(
tblSchema[i].name,
ncName.title,
col.type,
@ -736,7 +737,7 @@ export default async (
if (enableErrorLogs)
console.log(`## Invalid column IDs mapped; skip`);
addToSkipColumnLog(
updateMigrationSkipLog(
srcTableSchema.title,
aTblColumns[i].name,
aTblColumns[i].type,
@ -794,7 +795,7 @@ export default async (
const fTblField =
nestedLookupTbl[i].typeOptions.foreignTableRollupColumnId;
const name = aTbl_getColumnName(fTblField);
addToSkipColumnLog(
updateMigrationSkipLog(
ncSchema.tablesById[nestedLookupTbl[i].srcTableId]?.title,
nestedLookupTbl[i].name,
nestedLookupTbl[i].type,
@ -908,7 +909,7 @@ export default async (
);
if (ncRollupFn === '') {
addToSkipColumnLog(
updateMigrationSkipLog(
srcTableSchema.title,
aTblColumns[i].name,
aTblColumns[i].type,
@ -924,7 +925,7 @@ export default async (
if (enableErrorLogs)
console.log(`## Invalid column IDs mapped; skip`);
addToSkipColumnLog(
updateMigrationSkipLog(
srcTableSchema.title,
aTblColumns[i].name,
aTblColumns[i].type,
@ -946,6 +947,24 @@ export default async (
continue;
}
// skip, if rollup column was pointing to another virtual column
const ncColSchema = await nc_getColumnSchema(
aTblColumns[i].typeOptions.foreignTableRollupColumnId
);
if (
ncColSchema?.uidt === UITypes.Formula ||
ncColSchema?.uidt === UITypes.Lookup ||
ncColSchema?.uidt === UITypes.Rollup
) {
updateMigrationSkipLog(
srcTableSchema.title,
aTblColumns[i].name,
aTblColumns[i].type,
'rollup referring to a lookup column'
);
continue;
}
const ncName = nc_getSanitizedColumnName(
srcTableSchema,
aTblColumns[i].name
@ -1190,6 +1209,8 @@ export default async (
rec[key] = atDateField.utc().format('YYYY-MM-DD HH:mm');
}
if (dt === UITypes.MultiSelect) rec[key] = value.join(',');
if (dt === UITypes.Attachment) {
const tempArr = [];
for (const v of value) {
@ -1424,11 +1445,12 @@ export default async (
// response will not include form object if everything is default
//
if (vData.metadata?.form) {
refreshMode = vData.metadata.form.refreshAfterSubmit;
msg = vData.metadata.form?.afterSubmitMessage
? vData.metadata.form.afterSubmitMessage
: 'Thank you for submitting the form!';
desc = vData.metadata.form.description;
if (vData.metadata.form?.refreshAfterSubmit)
refreshMode = vData.metadata.form.refreshAfterSubmit;
if (vData.metadata.form?.afterSubmitMessage)
msg = vData.metadata.form.afterSubmitMessage;
if (vData.metadata.form?.description)
desc = vData.metadata.form.description;
}
const formData = {
@ -1490,7 +1512,7 @@ export default async (
x => x.id === gridViews[i].id
)?.name;
const viewList: any = await api.dbView.list(tblId);
const ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id;
let ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id;
// create view (default already created)
if (i > 0) {
@ -1506,6 +1528,7 @@ export default async (
tblId
);
// syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Create ${viewName}`)
ncViewId = viewCreated.id;
}
// syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Hide columns ${viewName}`)
@ -1724,8 +1747,11 @@ export default async (
// column not available;
// one of not migrated column;
if (!colSchema) {
addUserInfo(
`Filter configuration partial: ${sMap.getNcNameFromAtId(viewId)}`
updateMigrationSkipLog(
sMap.getNcNameFromAtId(viewId),
colSchema.title,
colSchema.uidt,
`filter config skipped; column not migrated`
);
continue;
}
@ -1737,8 +1763,11 @@ export default async (
// console.log(filter)
if (datatype === UITypes.Date) {
// skip filters over data datatype
addUserInfo(
`Filter configuration partial: ${sMap.getNcNameFromAtId(viewId)}`
updateMigrationSkipLog(
sMap.getNcNameFromAtId(viewId),
colSchema.title,
colSchema.uidt,
`filter config skipped; filter over date datatype not supported`
);
continue;
}
@ -1861,12 +1890,6 @@ export default async (
}
///////////////////////////////////////////////////////////////////////////////
const userInfo = [];
function addUserInfo(log) {
userInfo.push(log);
}
try {
syncLog = progress;
progress('SDK initialized');

6
scripts/cypress/cypress.json

@ -43,7 +43,11 @@
"user": "root",
"password": "password"
},
"screenshot": false
"screenshot": false,
"airtable": {
"apiKey": "keyn1MR87qgyUsYg4",
"sharedBase": "https://airtable.com/shrkSQdtKNzUfAbIY"
}
},
"fixturesFolder": "scripts/cypress/fixtures",
"integrationFolder": "scripts/cypress/integration",

62
scripts/cypress/integration/common/7b_import_from_airtable.js

@ -0,0 +1,62 @@
// Cypress test suite: Project import from Airtable
//
import { isTestSuiteActive } from "../../support/page_objects/projectConstants";
let apiKey = ""
let sharedBase = ""
export const genTest = (apiType, dbType) => {
if (!isTestSuiteActive(apiType, dbType)) return;
describe(`Import from airtable`, () => {
before(() => {
apiKey = Cypress.env("airtable").apiKey;
sharedBase = Cypress.env("airtable").sharedBase;
});
after(() => {});
it("Import", () => {
cy.log(apiKey, sharedBase);
// trigger import
cy.get(".add-btn").click();
cy.getActiveMenu().contains("Airtable").should('exist').click();
// enable turbo
// cy.getActiveModal().find(".nc-btn-enable-turbo").should('exist').click()
cy.getActiveModal().find(".nc-input-api-key").should('exist').clear().type(apiKey)
cy.getActiveModal().find(".nc-input-shared-base").should('exist').clear().type(sharedBase)
cy.getActiveModal().find(".nc-btn-airtable-import").should('exist').click()
// it will take a while for import to finish
cy.getActiveModal().find(".nc-btn-go-dashboard", {timeout: 120000}).should('exist').click()
});
it("Verify Schema", () => {});
it("Verify Data", () => {});
});
};
/**
* @copyright Copyright (c) 2021, Xgene Cloud Ltd
*
* @author Raju Udava <sivadstala@gmail.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
Loading…
Cancel
Save