Merge pull request #2175 from nocodb/feat/sync-options

Feat/sync UI options

packages/nc-gui/components/import/ImportFromAirtable.vue

@@ -21,7 +21,7 @@
           >
             <template v-if="step === 1">
               <div class="d-flex flex-column justify-center align-center pt-2 pb-6">
-                <span class="subtitle-1 font-weight-medium" @dblclick="$set(syncSource.details,'syncViews',true)">
+                <span class="subtitle-1 font-weight-medium" @dblclick="$set(syncSource.details.options,'syncViews',true)">
                   Credentials
                 </span>
 
@@ -55,7 +55,53 @@
                     :rules="[(v) => !!v || 'Shared Base ID / URL is required']"
                   />
                 </div>
-              </v-form>   <v-card-actions class="justify-center pb-6">
+                <v-expansion-panels class="mx-auto" style="width: 50%;" flat>
+                  <v-expansion-panel>
+                    <v-expansion-panel-header>Advanced Options</v-expansion-panel-header>
+                    <v-expansion-panel-content>
+                      <v-checkbox
+                        v-model="syncSource.details.options.syncData"
+                        label="Import Data"
+                        hide-details
+                        dense
+                      />
+                      <v-checkbox
+                        v-model="syncSource.details.options.syncRollup"
+                        label="Import Rollup Columns"
+                        hide-details
+                        dense
+                      />
+                      <v-checkbox
+                        v-model="syncSource.details.options.syncLookup"
+                        label="Import Lookup Columns"
+                        hide-details
+                        dense
+                      />
+                      <v-checkbox
+                        v-model="syncSource.details.options.syncAttachment"
+                        label="Import Attachment Columns"
+                        hide-details
+                        dense
+                      />
+                      <v-tooltip bottom>
+                        <template #activator="{ on }">
+                          <div v-on="on">
+                            <v-checkbox
+                              v-model="syncSource.details.options.syncFormula"
+                              label="Import Formula Columns"
+                              hide-details
+                              dense
+                              disabled
+                            />
+                          </div>
+                        </template>
+                        <span>Coming Soon!</span>
+                      </v-tooltip>
+                    </v-expansion-panel-content>
+                  </v-expansion-panel>
+                </v-expansion-panels>
+              </v-form>
+              <v-card-actions class="justify-center pb-6">
                 <v-btn
                   v-t="['c:sync-airtable:save-and-sync']"
                   class="nc-btn-airtable-import"
@@ -139,7 +185,24 @@ export default {
     socket: null,
     step: 1,
     progress: [],
-    syncSource: null,
+    syncSource: {
+      type: 'Airtable',
+      details: {
+        syncInterval: '15mins',
+        syncDirection: 'Airtable to NocoDB',
+        syncRetryCount: 1,
+        apiKey: '',
+        shareId: '',
+        options: {
+          syncViews: false,
+          syncData: true,
+          syncRollup: false,
+          syncLookup: true,
+          syncFormula: false,
+          syncAttachment: true
+        }
+      }
+    },
     syncSourceUrlOrId: ''
   }),
   computed: {
@@ -221,7 +284,7 @@ export default {
       const { data: { list: srcs } } = await this.$axios.get(`/api/v1/db/meta/projects/${this.projectId}/syncs`)
       if (srcs && srcs[0]) {
         srcs[0].details = srcs[0].details || {}
-        this.syncSource = srcs[0]
+        this.syncSource = this.migrateSync(srcs[0])
         this.syncSourceUrlOrId = srcs[0].details.shareId
       } else {
         this.syncSource = {
@@ -230,11 +293,16 @@ export default {
             syncInterval: '15mins',
             syncDirection: 'Airtable to NocoDB',
             syncRetryCount: 1,
-
-            syncViews: false,
-
             apiKey: '',
-            shareId: ''
+            shareId: '',
+            options: {
+              syncViews: false,
+              syncData: true,
+              syncRollup: false,
+              syncLookup: true,
+              syncFormula: false,
+              syncAttachment: true
+            }
           }
         }
       }
@@ -252,8 +320,23 @@ export default {
       }
     },
     enableTurbo() {
-      this.$set(this.syncSource.details, 'syncViews', true)
+      this.$set(this.syncSource.details.options, 'syncViews', true)
       this.$toast.success('🚀🚀 Ludicrous mode activated! Let\'s go! 🚀🚀').goAway(3000)
+    },
+    migrateSync(src) {
+      if (!src.details?.options) {
+        src.details.options = {
+          syncViews: false,
+          syncData: true,
+          syncRollup: false,
+          syncLookup: true,
+          syncFormula: false,
+          syncAttachment: true
+        }
+        src.details.options.syncViews = src.syncViews
+        delete src.syncViews
+      }
+      return src
     }
   }
 }

packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts

@@ -1,5 +1,6 @@
 import FetchAT from './fetchAT';
 import { UITypes } from 'nocodb-sdk';
+import { Tele } from 'nc-help';
 // import * as sMap from './syncMap';
 import FormData from 'form-data';
 
@@ -70,7 +71,6 @@ export default async (
   let base, baseId;
   const start = Date.now();
   const enableErrorLogs = false;
-  const process_aTblData = true;
   const generate_migrationStats = true;
   const debugMode = false;
   let api: Api<any>;
@@ -387,7 +387,7 @@ export default async (
     for (let i = 0; i < tblSchema.length; ++i) {
       const table: any = {};
 
-      if (syncDB.syncViews) {
+      if (syncDB.options.syncViews) {
         rtc.view.total += tblSchema[i].views.reduce(
           (acc, cur) =>
             ['grid', 'form', 'gallery'].includes(cur.type) ? ++acc : acc,
@@ -1506,7 +1506,7 @@ export default async (
   }
 
   async function nocoConfigureGalleryView(sDB, aTblSchema) {
-    if (!sDB.syncViews) return;
+    if (!sDB.options.syncViews) return;
     for (let idx = 0; idx < aTblSchema.length; idx++) {
       const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
       const galleryViews = aTblSchema[idx].views.filter(
@@ -1545,7 +1545,7 @@ export default async (
   }
 
   async function nocoConfigureFormView(sDB, aTblSchema) {
-    if (!sDB.syncViews) return;
+    if (!sDB.options.syncViews) return;
     for (let idx = 0; idx < aTblSchema.length; idx++) {
       const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
       const formViews = aTblSchema[idx].views.filter(x => x.type === 'form');
@@ -1623,11 +1623,11 @@ export default async (
       const gridViews = aTblSchema[idx].views.filter(x => x.type === 'grid');
 
       let viewCnt = idx;
-      if (syncDB.syncViews)
+      if (syncDB.options.syncViews)
         viewCnt = rtc.view.grid + rtc.view.gallery + rtc.view.form;
       rtc.view.grid += gridViews.length;
 
-      for (let i = 0; i < (sDB.syncViews ? gridViews.length : 1); i++) {
+      for (let i = 0; i < (sDB.options.syncViews ? gridViews.length : 1); i++) {
         logDetailed(`   Axios fetch view-data`);
         // fetch viewData JSON
         const vData = await getViewData(gridViews[i].id);
@@ -1870,6 +1870,32 @@ export default async (
         spaces: 2
       });
     }
+
+    Tele.event({
+      event: 'a:airtable-import:success',
+      data: {
+        stats: {
+          migrationTime: duration,
+          totalTables: aTblSchema.length,
+          totalColumns: columnSum,
+          links: linkSum,
+          lookup: lookupSum,
+          rollup: rollupSum,
+          totalFilters: rtc.filter,
+          totalSort: rtc.sort,
+          view: {
+            total: rtc.view.total,
+            grid: rtc.view.grid,
+            gallery: rtc.view.gallery,
+            form: rtc.view.form
+          },
+          axios: {
+            count: rtc.fetchAt.count,
+            time: rtc.fetchAt.time
+          }
+        }
+      }
+    });
   }
 
   //////////////////////////////
@@ -2113,21 +2139,26 @@ export default async (
     await nocoCreateLinkToAnotherRecord(aTblSchema);
     logDetailed('Migrating LTAR columns completed');
 
-    logDetailed(`Configuring Lookup`);
-    // add look-ups
-    await nocoCreateLookups(aTblSchema);
-    logDetailed('Migrating Lookup columns completed');
-
-    logDetailed('Configuring Rollup');
-    // add roll-ups
-    await nocoCreateRollup(aTblSchema);
-    logDetailed('Migrating Rollup columns completed');
-
-    logDetailed('Migrating Lookup form Rollup columns');
-    // lookups for rollup
-    await nocoLookupForRollup();
-    logDetailed('Migrating Lookup form Rollup columns completed');
+    if (syncDB.options.syncLookup) {
+      logDetailed(`Configuring Lookup`);
+      // add look-ups
+      await nocoCreateLookups(aTblSchema);
+      logDetailed('Migrating Lookup columns completed');
+    }
 
+    if (syncDB.options.syncRollup) {
+      logDetailed('Configuring Rollup');
+      // add roll-ups
+      await nocoCreateRollup(aTblSchema);
+      logDetailed('Migrating Rollup columns completed');
+
+      if (syncDB.options.syncLookup) {
+        logDetailed('Migrating Lookup form Rollup columns');
+        // lookups for rollup
+        await nocoLookupForRollup();
+        logDetailed('Migrating Lookup form Rollup columns completed');
+      }
+    }
     logDetailed('Configuring Primary value column');
     // configure primary values
     await nocoSetPrimary(aTblSchema);
@@ -2148,7 +2179,7 @@ export default async (
     await nocoConfigureGalleryView(syncDB, aTblSchema);
     logDetailed('Syncing views completed');
 
-    if (process_aTblData) {
+    if (syncDB.options.syncData) {
       try {
         // await nc_DumpTableSchema();
         const _perfStart = recordPerfStart();
@@ -2226,6 +2257,10 @@ export default async (
     }
   } catch (e) {
     if (e.response?.data?.msg) {
+      Tele.event({
+        event: 'a:airtable-import:error',
+        data: { error: e.response.data.msg }
+      });
       throw new Error(e.response.data.msg);
     }
     throw e;
@@ -2254,5 +2289,12 @@ export interface AirtableSyncConfig {
   projectId?: string;
   apiKey: string;
   shareId: string;
-  syncViews: boolean;
+  options: {
+    syncViews: boolean;
+    syncData: boolean;
+    syncRollup: boolean;
+    syncLookup: boolean;
+    syncFormula: boolean;
+    syncAttachment: boolean;
+  };
 }