Feat/at sync test: migration fixes (#2068)

* fix: ignore rollup for lookup columns

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* test/cypress: base file for import airtable verification

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* test/cypress: import access creds

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: multiselect as csv string instead of array

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: ludicrous mode view data

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: use decimal if precision configured, support for default form view

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: handle ? in column name

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* refactor: migration logs

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

packages/nc-gui/components/import/ImportFromAirtable.vue

@@ -5,7 +5,7 @@
         <h3 class="mt-2">
           {{ $t('title.importFromAirtable') }}
         </h3>
-        <div v-t="['c:airtable-import:turbo-mode']" class="ml-2 mt-3 title pointer" @click="enableTurbo">
+        <div v-t="['c:airtable-import:turbo-mode']" class="ml-2 mt-3 title pointer nc-btn-enable-turbo" @click="enableTurbo">
           🚀
         </div>
         <v-spacer />
@@ -30,7 +30,7 @@
                   outlined
                   dense
                   label="Api Key"
-                  class="caption"
+                  class="caption nc-input-api-key"
                   :type="isPasswordVisible ? 'text':'password'"
                   :rules="[v=> !!v || 'Api Key is required']"
                 >
@@ -45,13 +45,14 @@
                   outlined
                   dense
                   label="Shared Base ID / URL"
-                  class="caption"
+                  class="caption nc-input-shared-base"
                   :rules="[(v) => !!v || 'Shared Base ID / URL is required']"
                 />
               </div>
             </v-form>   <v-card-actions class="justify-center pb-6">
               <v-btn
                 v-t="['c:sync-airtable:save-and-sync']"
+                class="nc-btn-airtable-import"
                 :disabled="!valid"
                 large
                 color="primary"
@@ -103,7 +104,7 @@
               v-if="progress && progress.length && progress[progress.length-1].status === 'COMPLETED'"
               class="pa-4 pt-8 text-center"
             >
-              <v-btn large color="primary" @click="airtableModal=false">
+              <v-btn large color="primary" class="nc-btn-go-dashboard" @click="airtableModal=false">
                 Go to dashboard
               </v-btn>
             </div>

packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts

@@ -1188,7 +1188,7 @@ class BaseModelSqlv2 {
           break;
         default:
           res[
-            column.title || column.column_name
+            (column.title || column.column_name).replace(/\?/g, '\\?')
           ] = `${this.model.table_name}.${column.column_name}`;
           break;
       }

packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts

@@ -73,15 +73,15 @@ export default async (
       count: 0,
       time: 0
     },
-    columnNotMigrated: {
+    migrationSkipLog: {
       count: 0,
       log: []
     }
   };
 
-  function addToSkipColumnLog(tbl, col, type, reason?) {
+  function updateMigrationSkipLog(tbl, col, type, reason?) {
-    runTimeCounters.columnNotMigrated.count++;
+    runTimeCounters.migrationSkipLog.count++;
-    runTimeCounters.columnNotMigrated.log.push(
+    runTimeCounters.migrationSkipLog.log.push(
       `tn[${tbl}] cn[${col}] type[${type}] :: ${reason}`
     );
   }
@@ -305,6 +305,7 @@ export default async (
           ncType = UITypes.Duration;
         else if (col.typeOptions?.format === 'currency')
           ncType = UITypes.Currency;
+        else if (col.typeOptions?.precision > 0) ncType = UITypes.Decimal;
         break;
 
       case 'formula':
@@ -411,7 +412,7 @@ export default async (
         // not supported datatype: pure formula field
         // allow formula based computed fields (created time/ modified time to go through)
         if (ncCol.uidt === UITypes.Formula) {
-          addToSkipColumnLog(
+          updateMigrationSkipLog(
             tblSchema[i].name,
             ncName.title,
             col.type,
@@ -736,7 +737,7 @@ export default async (
             if (enableErrorLogs)
               console.log(`## Invalid column IDs mapped; skip`);
 
-            addToSkipColumnLog(
+            updateMigrationSkipLog(
               srcTableSchema.title,
               aTblColumns[i].name,
               aTblColumns[i].type,
@@ -794,7 +795,7 @@ export default async (
           const fTblField =
             nestedLookupTbl[i].typeOptions.foreignTableRollupColumnId;
           const name = aTbl_getColumnName(fTblField);
-          addToSkipColumnLog(
+          updateMigrationSkipLog(
             ncSchema.tablesById[nestedLookupTbl[i].srcTableId]?.title,
             nestedLookupTbl[i].name,
             nestedLookupTbl[i].type,
@@ -908,7 +909,7 @@ export default async (
           );
 
           if (ncRollupFn === '') {
-            addToSkipColumnLog(
+            updateMigrationSkipLog(
               srcTableSchema.title,
               aTblColumns[i].name,
               aTblColumns[i].type,
@@ -924,7 +925,7 @@ export default async (
             if (enableErrorLogs)
               console.log(`## Invalid column IDs mapped; skip`);
 
-            addToSkipColumnLog(
+            updateMigrationSkipLog(
               srcTableSchema.title,
               aTblColumns[i].name,
               aTblColumns[i].type,
@@ -946,6 +947,24 @@ export default async (
             continue;
           }
 
+          // skip, if rollup column was pointing to another virtual column
+          const ncColSchema = await nc_getColumnSchema(
+            aTblColumns[i].typeOptions.foreignTableRollupColumnId
+          );
+          if (
+            ncColSchema?.uidt === UITypes.Formula ||
+            ncColSchema?.uidt === UITypes.Lookup ||
+            ncColSchema?.uidt === UITypes.Rollup
+          ) {
+            updateMigrationSkipLog(
+              srcTableSchema.title,
+              aTblColumns[i].name,
+              aTblColumns[i].type,
+              'rollup referring to a lookup column'
+            );
+            continue;
+          }
+
           const ncName = nc_getSanitizedColumnName(
             srcTableSchema,
             aTblColumns[i].name
@@ -1190,6 +1209,8 @@ export default async (
         rec[key] = atDateField.utc().format('YYYY-MM-DD HH:mm');
       }
 
+      if (dt === UITypes.MultiSelect) rec[key] = value.join(',');
+
       if (dt === UITypes.Attachment) {
         const tempArr = [];
         for (const v of value) {
@@ -1424,10 +1445,11 @@ export default async (
         // response will not include form object if everything is default
         //
         if (vData.metadata?.form) {
+          if (vData.metadata.form?.refreshAfterSubmit)
             refreshMode = vData.metadata.form.refreshAfterSubmit;
-          msg = vData.metadata.form?.afterSubmitMessage
+          if (vData.metadata.form?.afterSubmitMessage)
-            ? vData.metadata.form.afterSubmitMessage
+            msg = vData.metadata.form.afterSubmitMessage;
-            : 'Thank you for submitting the form!';
+          if (vData.metadata.form?.description)
             desc = vData.metadata.form.description;
         }
 
@@ -1490,7 +1512,7 @@ export default async (
           x => x.id === gridViews[i].id
         )?.name;
         const viewList: any = await api.dbView.list(tblId);
-        const ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id;
+        let ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id;
 
         // create view (default already created)
         if (i > 0) {
@@ -1506,6 +1528,7 @@ export default async (
             tblId
           );
           // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Create ${viewName}`)
+          ncViewId = viewCreated.id;
         }
 
         // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Hide columns ${viewName}`)
@@ -1724,8 +1747,11 @@ export default async (
       // column not available;
       // one of not migrated column;
       if (!colSchema) {
-        addUserInfo(
+        updateMigrationSkipLog(
-          `Filter configuration partial: ${sMap.getNcNameFromAtId(viewId)}`
+          sMap.getNcNameFromAtId(viewId),
+          colSchema.title,
+          colSchema.uidt,
+          `filter config skipped; column not migrated`
         );
         continue;
       }
@@ -1737,8 +1763,11 @@ export default async (
       // console.log(filter)
       if (datatype === UITypes.Date) {
         // skip filters over data datatype
-        addUserInfo(
+        updateMigrationSkipLog(
-          `Filter configuration partial: ${sMap.getNcNameFromAtId(viewId)}`
+          sMap.getNcNameFromAtId(viewId),
+          colSchema.title,
+          colSchema.uidt,
+          `filter config skipped; filter over date datatype not supported`
         );
         continue;
       }
@@ -1861,12 +1890,6 @@ export default async (
   }
 
   ///////////////////////////////////////////////////////////////////////////////
-  const userInfo = [];
-
-  function addUserInfo(log) {
-    userInfo.push(log);
-  }
-
   try {
     syncLog = progress;
     progress('SDK initialized');

scripts/cypress/cypress.json

@@ -43,7 +43,11 @@
             "user": "root",
             "password": "password"
         },
-        "screenshot": false
+        "screenshot": false,
+        "airtable": {
+            "apiKey": "keyn1MR87qgyUsYg4",
+            "sharedBase": "https://airtable.com/shrkSQdtKNzUfAbIY"
+        }
     },
     "fixturesFolder": "scripts/cypress/fixtures",
     "integrationFolder": "scripts/cypress/integration",

scripts/cypress/integration/common/7b_import_from_airtable.js

@@ -0,0 +1,62 @@
+// Cypress test suite: Project import from Airtable
+//
+
+import { isTestSuiteActive } from "../../support/page_objects/projectConstants";
+let apiKey = ""
+let sharedBase = ""
+
+export const genTest = (apiType, dbType) => {
+  if (!isTestSuiteActive(apiType, dbType)) return;
+
+  describe(`Import from airtable`, () => {
+    before(() => {
+      apiKey = Cypress.env("airtable").apiKey;
+      sharedBase = Cypress.env("airtable").sharedBase;
+    });
+
+    after(() => {});
+
+    it("Import", () => {
+      cy.log(apiKey, sharedBase);
+
+      // trigger import
+      cy.get(".add-btn").click();
+      cy.getActiveMenu().contains("Airtable").should('exist').click();
+
+      // enable turbo
+      // cy.getActiveModal().find(".nc-btn-enable-turbo").should('exist').click()
+
+      cy.getActiveModal().find(".nc-input-api-key").should('exist').clear().type(apiKey)
+      cy.getActiveModal().find(".nc-input-shared-base").should('exist').clear().type(sharedBase)
+      cy.getActiveModal().find(".nc-btn-airtable-import").should('exist').click()
+
+      // it will take a while for import to finish
+      cy.getActiveModal().find(".nc-btn-go-dashboard", {timeout: 120000}).should('exist').click()
+
+    });
+    it("Verify Schema", () => {});
+    it("Verify Data", () => {});
+  });
+};
+
+/**
+ * @copyright Copyright (c) 2021, Xgene Cloud Ltd
+ *
+ * @author Raju Udava <sivadstala@gmail.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */