github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Merge pull request #2175 from nocodb/feat/sync-options 

Feat/sync UI options
pull/2178/head
Raju Udava 2 years ago committed by GitHub
parent
f12410aa33 d7bd9a6c23
commit
fa7771fbc9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 156 additions and 31 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 101
      packages/nc-gui/components/import/ImportFromAirtable.vue
  2. 86
      packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts

101
packages/nc-gui/components/import/ImportFromAirtable.vue
Unescape View File

@ -21,7 +21,7 @@
> >
<template v-if="step === 1"> <template v-if="step === 1">
<div class="d-flex flex-column justify-center align-center pt-2 pb-6"> <div class="d-flex flex-column justify-center align-center pt-2 pb-6">
<span class="subtitle-1 font-weight-medium" @dblclick="$set(syncSource.details,'syncViews',true)"> <span class="subtitle-1 font-weight-medium" @dblclick="$set(syncSource.details.options,'syncViews',true)">
Credentials Credentials
</span> </span>
@ -55,7 +55,53 @@
:rules="[(v) => !!v || 'Shared Base ID / URL is required']" :rules="[(v) => !!v || 'Shared Base ID / URL is required']"
/> />
</div> </div>
</v-form> <v-card-actions class="justify-center pb-6"> <v-expansion-panels class="mx-auto" style="width: 50%;" flat>
<v-expansion-panel>
<v-expansion-panel-header>Advanced Options</v-expansion-panel-header>
<v-expansion-panel-content>
<v-checkbox
v-model="syncSource.details.options.syncData"
label="Import Data"
hide-details
dense
/>
<v-checkbox
v-model="syncSource.details.options.syncRollup"
label="Import Rollup Columns"
hide-details
dense
/>
<v-checkbox
v-model="syncSource.details.options.syncLookup"
label="Import Lookup Columns"
hide-details
dense
/>
<v-checkbox
v-model="syncSource.details.options.syncAttachment"
label="Import Attachment Columns"
hide-details
dense
/>
<v-tooltip bottom>
<template #activator="{ on }">
<div v-on="on">
<v-checkbox
v-model="syncSource.details.options.syncFormula"
label="Import Formula Columns"
hide-details
dense
disabled
/>
</div>
</template>
<span>Coming Soon!</span>
</v-tooltip>
</v-expansion-panel-content>
</v-expansion-panel>
</v-expansion-panels>
</v-form>
<v-card-actions class="justify-center pb-6">
<v-btn <v-btn
v-t="['c:sync-airtable:save-and-sync']" v-t="['c:sync-airtable:save-and-sync']"
class="nc-btn-airtable-import" class="nc-btn-airtable-import"
@ -139,7 +185,24 @@ export default {
socket: null, socket: null,
step: 1, step: 1,
progress: [], progress: [],
syncSource: null, syncSource: {
type: 'Airtable',
details: {
syncInterval: '15mins',
syncDirection: 'Airtable to NocoDB',
syncRetryCount: 1,
apiKey: '',
shareId: '',
options: {
syncViews: false,
syncData: true,
syncRollup: false,
syncLookup: true,
syncFormula: false,
syncAttachment: true
}
}
},
syncSourceUrlOrId: '' syncSourceUrlOrId: ''
}), }),
computed: { computed: {
@ -221,7 +284,7 @@ export default {
const { data: { list: srcs } } = await this.$axios.get(`/api/v1/db/meta/projects/${this.projectId}/syncs`) const { data: { list: srcs } } = await this.$axios.get(`/api/v1/db/meta/projects/${this.projectId}/syncs`)
if (srcs && srcs[0]) { if (srcs && srcs[0]) {
srcs[0].details = srcs[0].details || {} srcs[0].details = srcs[0].details || {}
this.syncSource = srcs[0] this.syncSource = this.migrateSync(srcs[0])
this.syncSourceUrlOrId = srcs[0].details.shareId this.syncSourceUrlOrId = srcs[0].details.shareId
} else { } else {
this.syncSource = { this.syncSource = {
@ -230,11 +293,16 @@ export default {
syncInterval: '15mins', syncInterval: '15mins',
syncDirection: 'Airtable to NocoDB', syncDirection: 'Airtable to NocoDB',
syncRetryCount: 1, syncRetryCount: 1,
syncViews: false,
apiKey: '', apiKey: '',
shareId: '' shareId: '',
options: {
syncViews: false,
syncData: true,
syncRollup: false,
syncLookup: true,
syncFormula: false,
syncAttachment: true
}
} }
} }
} }
@ -252,8 +320,23 @@ export default {
} }
}, },
enableTurbo() { enableTurbo() {
this.$set(this.syncSource.details, 'syncViews', true) this.$set(this.syncSource.details.options, 'syncViews', true)
this.$toast.success('🚀🚀 Ludicrous mode activated! Let\'s go! 🚀🚀').goAway(3000) this.$toast.success('🚀🚀 Ludicrous mode activated! Let\'s go! 🚀🚀').goAway(3000)
},
migrateSync(src) {
if (!src.details?.options) {
src.details.options = {
syncViews: false,
syncData: true,
syncRollup: false,
syncLookup: true,
syncFormula: false,
syncAttachment: true
}
src.details.options.syncViews = src.syncViews
delete src.syncViews
}
return src
} }
} }
} }

86
packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts
Unescape View File

@ -1,5 +1,6 @@
import FetchAT from './fetchAT'; import FetchAT from './fetchAT';
import { UITypes } from 'nocodb-sdk'; import { UITypes } from 'nocodb-sdk';
import { Tele } from 'nc-help';
// import * as sMap from './syncMap'; // import * as sMap from './syncMap';
import FormData from 'form-data'; import FormData from 'form-data';
@ -70,7 +71,6 @@ export default async (
let base, baseId; let base, baseId;
const start = Date.now(); const start = Date.now();
const enableErrorLogs = false; const enableErrorLogs = false;
const process_aTblData = true;
const generate_migrationStats = true; const generate_migrationStats = true;
const debugMode = false; const debugMode = false;
let api: Api<any>; let api: Api<any>;
@ -387,7 +387,7 @@ export default async (
for (let i = 0; i < tblSchema.length; ++i) { for (let i = 0; i < tblSchema.length; ++i) {
const table: any = {}; const table: any = {};
if (syncDB.syncViews) { if (syncDB.options.syncViews) {
rtc.view.total += tblSchema[i].views.reduce( rtc.view.total += tblSchema[i].views.reduce(
(acc, cur) => (acc, cur) =>
['grid', 'form', 'gallery'].includes(cur.type) ? ++acc : acc, ['grid', 'form', 'gallery'].includes(cur.type) ? ++acc : acc,
@ -1506,7 +1506,7 @@ export default async (
} }
async function nocoConfigureGalleryView(sDB, aTblSchema) { async function nocoConfigureGalleryView(sDB, aTblSchema) {
if (!sDB.syncViews) return; if (!sDB.options.syncViews) return;
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id; const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
const galleryViews = aTblSchema[idx].views.filter( const galleryViews = aTblSchema[idx].views.filter(
@ -1545,7 +1545,7 @@ export default async (
} }
async function nocoConfigureFormView(sDB, aTblSchema) { async function nocoConfigureFormView(sDB, aTblSchema) {
if (!sDB.syncViews) return; if (!sDB.options.syncViews) return;
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
const formViews = aTblSchema[idx].views.filter(x => x.type === 'form'); const formViews = aTblSchema[idx].views.filter(x => x.type === 'form');
@ -1623,11 +1623,11 @@ export default async (
const gridViews = aTblSchema[idx].views.filter(x => x.type === 'grid'); const gridViews = aTblSchema[idx].views.filter(x => x.type === 'grid');
let viewCnt = idx; let viewCnt = idx;
if (syncDB.syncViews) if (syncDB.options.syncViews)
viewCnt = rtc.view.grid + rtc.view.gallery + rtc.view.form; viewCnt = rtc.view.grid + rtc.view.gallery + rtc.view.form;
rtc.view.grid += gridViews.length; rtc.view.grid += gridViews.length;
for (let i = 0; i < (sDB.syncViews ? gridViews.length : 1); i++) { for (let i = 0; i < (sDB.options.syncViews ? gridViews.length : 1); i++) {
logDetailed(` Axios fetch view-data`); logDetailed(` Axios fetch view-data`);
// fetch viewData JSON // fetch viewData JSON
const vData = await getViewData(gridViews[i].id); const vData = await getViewData(gridViews[i].id);
@ -1870,6 +1870,32 @@ export default async (
spaces: 2 spaces: 2
}); });
} }
Tele.event({
event: 'a:airtable-import:success',
data: {
stats: {
migrationTime: duration,
totalTables: aTblSchema.length,
totalColumns: columnSum,
links: linkSum,
lookup: lookupSum,
rollup: rollupSum,
totalFilters: rtc.filter,
totalSort: rtc.sort,
view: {
total: rtc.view.total,
grid: rtc.view.grid,
gallery: rtc.view.gallery,
form: rtc.view.form
},
axios: {
count: rtc.fetchAt.count,
time: rtc.fetchAt.time
}
}
}
});
} }
////////////////////////////// //////////////////////////////
@ -2113,21 +2139,26 @@ export default async (
await nocoCreateLinkToAnotherRecord(aTblSchema); await nocoCreateLinkToAnotherRecord(aTblSchema);
logDetailed('Migrating LTAR columns completed'); logDetailed('Migrating LTAR columns completed');
logDetailed(`Configuring Lookup`); if (syncDB.options.syncLookup) {
// add look-ups logDetailed(`Configuring Lookup`);
await nocoCreateLookups(aTblSchema); // add look-ups
logDetailed('Migrating Lookup columns completed'); await nocoCreateLookups(aTblSchema);
logDetailed('Migrating Lookup columns completed');
logDetailed('Configuring Rollup'); }
// add roll-ups
await nocoCreateRollup(aTblSchema);
logDetailed('Migrating Rollup columns completed');
logDetailed('Migrating Lookup form Rollup columns');
// lookups for rollup
await nocoLookupForRollup();
logDetailed('Migrating Lookup form Rollup columns completed');
if (syncDB.options.syncRollup) {
logDetailed('Configuring Rollup');
// add roll-ups
await nocoCreateRollup(aTblSchema);
logDetailed('Migrating Rollup columns completed');
if (syncDB.options.syncLookup) {
logDetailed('Migrating Lookup form Rollup columns');
// lookups for rollup
await nocoLookupForRollup();
logDetailed('Migrating Lookup form Rollup columns completed');
}
}
logDetailed('Configuring Primary value column'); logDetailed('Configuring Primary value column');
// configure primary values // configure primary values
await nocoSetPrimary(aTblSchema); await nocoSetPrimary(aTblSchema);
@ -2148,7 +2179,7 @@ export default async (
await nocoConfigureGalleryView(syncDB, aTblSchema); await nocoConfigureGalleryView(syncDB, aTblSchema);
logDetailed('Syncing views completed'); logDetailed('Syncing views completed');
if (process_aTblData) { if (syncDB.options.syncData) {
try { try {
// await nc_DumpTableSchema(); // await nc_DumpTableSchema();
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
@ -2226,6 +2257,10 @@ export default async (
} }
} catch (e) { } catch (e) {
if (e.response?.data?.msg) { if (e.response?.data?.msg) {
Tele.event({
event: 'a:airtable-import:error',
data: { error: e.response.data.msg }
});
throw new Error(e.response.data.msg); throw new Error(e.response.data.msg);
} }
throw e; throw e;
@ -2254,5 +2289,12 @@ export interface AirtableSyncConfig {
projectId?: string; projectId?: string;
apiKey: string; apiKey: string;
shareId: string; shareId: string;
syncViews: boolean; options: {
syncViews: boolean;
syncData: boolean;
syncRollup: boolean;
syncLookup: boolean;
syncFormula: boolean;
syncAttachment: boolean;
};
} }

Write Preview
Loading…
Cancel
Save