From 194b205a921e0126eb9c7438fae39a120f5622c1 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sun, 15 May 2022 11:14:35 +0530 Subject: [PATCH] Feat - Airtable import (#2048) * wip: add job manager Signed-off-by: Pranav C * wip: job manager impl Signed-off-by: Pranav C * wip: migrations Signed-off-by: Pranav C * wip Signed-off-by: Pranav C * wip: data sync Signed-off-by: Pranav C * feat: sync source creation Signed-off-by: Pranav C * feat: api integration Signed-off-by: Pranav C * chore: remove unused files Signed-off-by: Pranav C * fix: update to work with project id, handle exceptions Signed-off-by: Pranav C * feat: show more progress details Signed-off-by: Pranav C * fix: extract id from source creation api response Signed-off-by: Pranav C * feat: bring latest changes Signed-off-by: Pranav C * fix: column creation and data sync Signed-off-by: Pranav C * refactor: ui improvements Signed-off-by: Pranav C * refactor: bring changes from sync branch Signed-off-by: Pranav C * refactor: avoid opening additional socket Signed-off-by: Pranav C * chore: update package-lock Signed-off-by: Pranav C * fix: wait until data and LTAR insertion completes Signed-off-by: Pranav C * feat: load table list after sync completes Signed-off-by: Pranav C * enhancement: add navigation back to dashboard Signed-off-by: Pranav C * fix: column order as in base, column visibility clean-up Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * refactor: ui updates Signed-off-by: Pranav C * enhancement: load first table after sync Signed-off-by: Pranav C * refactor: common routine to sanitize column names Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * enhancement: accept both hared base id / url Signed-off-by: Pranav C * refactor: sanitize table name Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * sync: fetch improvements Signed-off-by: mertmit * sync: fix nestedLookup Signed-off-by: mertmit * fix: dateTime datatype support Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * enhancement: add validation for credential form Signed-off-by: Pranav C * fix: migrate default values if configured Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * fix: include support for created time, modified time Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * fix: rollup column function map Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * feat: log column not migrated info Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * feat: fetch template Signed-off-by: mertmit * fix: move record id & hash to bottom of list Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * refactor: add import from airtable under project tabs menu Signed-off-by: Pranav C Co-authored-by: Raju Udava <86527202+dstala@users.noreply.github.com> Co-authored-by: mertmit --- .../nc-gui/components/ProjectTreeView.vue | 94 +- .../components/import/importFromAirtable.vue | 281 +++ .../project/spreadsheet/public/xcKanban.vue | 1 + packages/nc-gui/components/projectTabs.vue | 27 +- .../components/settings/settingsModal.vue | 6 +- packages/nc-gui/lang/da.json | 3 +- packages/nc-gui/lang/de.json | 3 +- packages/nc-gui/lang/en.json | 3 +- packages/nc-gui/lang/es.json | 3 +- packages/nc-gui/lang/fa.json | 3 +- packages/nc-gui/lang/fi.json | 3 +- packages/nc-gui/lang/fr.json | 3 +- packages/nc-gui/lang/hr.json | 3 +- packages/nc-gui/lang/id.json | 3 +- packages/nc-gui/lang/it_IT.json | 3 +- packages/nc-gui/lang/iw.json | 3 +- packages/nc-gui/lang/ja.json | 3 +- packages/nc-gui/lang/ko.json | 3 +- packages/nc-gui/lang/lv.json | 3 +- packages/nc-gui/lang/nl.json | 3 +- packages/nc-gui/lang/no.json | 3 +- packages/nc-gui/lang/pl.json | 3 +- packages/nc-gui/lang/pt.json | 3 +- packages/nc-gui/lang/pt_BR.json | 3 +- packages/nc-gui/lang/ru.json | 3 +- packages/nc-gui/lang/sl.json | 3 +- packages/nc-gui/lang/sv.json | 3 +- packages/nc-gui/lang/th.json | 3 +- packages/nc-gui/lang/tr.json | 3 +- packages/nc-gui/lang/uk.json | 3 +- packages/nc-gui/lang/vi.json | 3 +- packages/nc-gui/lang/zh_CN.json | 3 +- packages/nc-gui/lang/zh_HK.json | 3 +- packages/nc-gui/lang/zh_TW.json | 3 +- packages/nc-gui/store/tabs.js | 14 + packages/nocodb/package-lock.json | 339 ++- packages/nocodb/package.json | 3 + .../src/lib/noco-jobs/EmitteryJobsMgr.ts | 35 + packages/nocodb/src/lib/noco-jobs/JobsMgr.ts | 65 + packages/nocodb/src/lib/noco-jobs/NocoJobs.ts | 20 + .../nocodb/src/lib/noco-jobs/RedisJobsMgr.ts | 54 + .../nocodb/src/lib/noco-models/Project.ts | 4 +- .../nocodb/src/lib/noco-models/SyncLogs.ts | 57 + .../nocodb/src/lib/noco-models/SyncSource.ts | 135 ++ .../lib/noco/common/XcMigrationSourcev2.ts | 9 +- .../nocodb/src/lib/noco/meta/api/index.ts | 10 +- .../api/sync/helpers/NocoSyncDestAdapter.ts | 6 + .../api/sync/helpers/NocoSyncSourceAdapter.ts | 7 + .../lib/noco/meta/api/sync/helpers/fetchAT.ts | 170 ++ .../src/lib/noco/meta/api/sync/helpers/job.ts | 2007 +++++++++++++++++ .../lib/noco/meta/api/sync/helpers/syncMap.ts | 31 + .../src/lib/noco/meta/api/sync/importApis.ts | 79 + .../lib/noco/meta/api/sync/syncSourceApis.ts | 53 + .../src/lib/noco/meta/helpers/extractProps.ts | 1 + .../noco/migrationsv2/nc_013_sync_source.ts | 72 + packages/nocodb/src/lib/utils/globals.ts | 4 +- 56 files changed, 3575 insertions(+), 96 deletions(-) create mode 100644 packages/nc-gui/components/import/importFromAirtable.vue create mode 100644 packages/nocodb/src/lib/noco-jobs/EmitteryJobsMgr.ts create mode 100644 packages/nocodb/src/lib/noco-jobs/JobsMgr.ts create mode 100644 packages/nocodb/src/lib/noco-jobs/NocoJobs.ts create mode 100644 packages/nocodb/src/lib/noco-jobs/RedisJobsMgr.ts create mode 100644 packages/nocodb/src/lib/noco-models/SyncLogs.ts create mode 100644 packages/nocodb/src/lib/noco-models/SyncSource.ts create mode 100644 packages/nocodb/src/lib/noco/meta/api/sync/helpers/NocoSyncDestAdapter.ts create mode 100644 packages/nocodb/src/lib/noco/meta/api/sync/helpers/NocoSyncSourceAdapter.ts create mode 100644 packages/nocodb/src/lib/noco/meta/api/sync/helpers/fetchAT.ts create mode 100644 packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts create mode 100644 packages/nocodb/src/lib/noco/meta/api/sync/helpers/syncMap.ts create mode 100644 packages/nocodb/src/lib/noco/meta/api/sync/importApis.ts create mode 100644 packages/nocodb/src/lib/noco/meta/api/sync/syncSourceApis.ts create mode 100644 packages/nocodb/src/lib/noco/migrationsv2/nc_013_sync_source.ts diff --git a/packages/nc-gui/components/ProjectTreeView.vue b/packages/nc-gui/components/ProjectTreeView.vue index f074bd2c27..865b086a80 100644 --- a/packages/nc-gui/components/ProjectTreeView.vue +++ b/packages/nc-gui/components/ProjectTreeView.vue @@ -17,7 +17,7 @@ > {{ $store.getters["project/GtrProjectName"] }} - + + v-if="item.children && item.children.length" + > ({{ - item.children.filter( - (child) => - !search || - child.name - .toLowerCase() - .includes(search.toLowerCase()) - ).length - }}) + item.children.filter( + (child) => + !search || + child.name + .toLowerCase() + .includes(search.toLowerCase()) + ).length + }}) + v-if="item.children && item.children.length" + > ({{ - item.children.filter( - (child) => - !search || - child.name - .toLowerCase() - .includes(search.toLowerCase()) - ).length - }}) + item.children.filter( + (child) => + !search || + child.name + .toLowerCase() + .includes(search.toLowerCase()) + ).length + }}) {{ item.name }} - + @@ -342,13 +342,13 @@ {{ - child.creator_tooltip - }} + child.creator_tooltip + }} {{ child.name }} @@ -611,8 +611,8 @@ {{ - $t("title.audit") - }} + $t("title.audit") + }} @@ -621,14 +621,14 @@ - + {{ - $t("activity.previewAs") - }} + $t("activity.previewAs") + }} mdi-drama-masks @@ -670,17 +670,15 @@ {{ - $t("activity.resetReview") - }} + $t("activity.resetReview") + }} - - - +
- - - +
diff --git a/packages/nc-gui/components/import/importFromAirtable.vue b/packages/nc-gui/components/import/importFromAirtable.vue new file mode 100644 index 0000000000..5da2803005 --- /dev/null +++ b/packages/nc-gui/components/import/importFromAirtable.vue @@ -0,0 +1,281 @@ + + + + + diff --git a/packages/nc-gui/components/project/spreadsheet/public/xcKanban.vue b/packages/nc-gui/components/project/spreadsheet/public/xcKanban.vue index dad14f9ab0..2fc1811673 100644 --- a/packages/nc-gui/components/project/spreadsheet/public/xcKanban.vue +++ b/packages/nc-gui/components/project/spreadsheet/public/xcKanban.vue @@ -102,6 +102,7 @@ export default { selectedExpandRowIndex: null, selectedExpandRowMeta: null, meta: null, + navDrawer: true, selected: { row: null, diff --git a/packages/nc-gui/components/projectTabs.vue b/packages/nc-gui/components/projectTabs.vue index 054114b3e7..d272c266b2 100644 --- a/packages/nc-gui/components/projectTabs.vue +++ b/packages/nc-gui/components/projectTabs.vue @@ -244,7 +244,7 @@ - + @@ -366,9 +383,11 @@ import GrpcClient from '@/components/project/grpcClient' import GlobalAcl from '@/components/globalAcl' import AuditTab from '~/components/project/auditTab' import QuickImport from '@/components/import/quickImport' +import ImportFromAirtable from '~/components/import/importFromAirtable' export default { components: { + ImportFromAirtable, SwaggerClient, // Screensaver, DlgTableCreate, @@ -407,7 +426,8 @@ export default { hideLogWindows: false, showScreensaver: false, quickImportModal: false, - quickImportType: '' + quickImportType: '', + airtableImportModal: false } }, methods: { @@ -489,6 +509,9 @@ export default { onImportFromExcelOrCSV(quickImportType) { this.quickImportModal = true this.quickImportType = quickImportType + }, + onAirtableImport() { + this.airtableImportModal = true } }, computed: { diff --git a/packages/nc-gui/components/settings/settingsModal.vue b/packages/nc-gui/components/settings/settingsModal.vue index e66179a0bc..93e4987dba 100644 --- a/packages/nc-gui/components/settings/settingsModal.vue +++ b/packages/nc-gui/components/settings/settingsModal.vue @@ -177,13 +177,15 @@ import DisableOrEnableModels from '~/components/project/projectMetadata/disableO import AuthTab from '~/components/authTab' import XcMeta from '~/components/project/settings/xcMeta' import AuditTab from '~/components/project/auditTab' +import ImportFromAirtable from '~/components/import/importFromAirtable' export default { name: 'SettingsModal', - components: { AuditTab, XcMeta, AuthTab, DisableOrEnableModels, AppStore }, + components: { ImportFromAirtable, AuditTab, XcMeta, AuthTab, DisableOrEnableModels, AppStore }, data: () => ({ settingsModal: false, - activePage: 'role' + activePage: 'role', + improtFromAirtableModal: false }) } diff --git a/packages/nc-gui/lang/da.json b/packages/nc-gui/lang/da.json index 6897015229..ac48c94ae5 100644 --- a/packages/nc-gui/lang/da.json +++ b/packages/nc-gui/lang/da.json @@ -171,7 +171,8 @@ "headLogin": "Log ind | Nocodb.", "resetPassword": "Nulstil din adgangskode", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Bemærk Via.", diff --git a/packages/nc-gui/lang/de.json b/packages/nc-gui/lang/de.json index eb1002ae60..3a0d47aad7 100644 --- a/packages/nc-gui/lang/de.json +++ b/packages/nc-gui/lang/de.json @@ -171,7 +171,8 @@ "headLogin": "Anmelden | NocoDB", "resetPassword": "Passwort zurücksetzen", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Benachrichtigen mit", diff --git a/packages/nc-gui/lang/en.json b/packages/nc-gui/lang/en.json index f9b21a4096..363d371b3a 100644 --- a/packages/nc-gui/lang/en.json +++ b/packages/nc-gui/lang/en.json @@ -171,7 +171,8 @@ "headLogin": "Log In | NocoDB", "resetPassword": "Reset your password", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Notify Via", diff --git a/packages/nc-gui/lang/es.json b/packages/nc-gui/lang/es.json index 0ca89fbf5b..22a6e992ab 100644 --- a/packages/nc-gui/lang/es.json +++ b/packages/nc-gui/lang/es.json @@ -171,7 +171,8 @@ "headLogin": "Acceder | NocoDB", "resetPassword": "Cambiar contraseña", "teamAndSettings": "Equipo y configuración", - "apiDocs": "Documentación de la API" + "apiDocs": "Documentación de la API", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Notificar a través de", diff --git a/packages/nc-gui/lang/fa.json b/packages/nc-gui/lang/fa.json index e62b177e34..d1b24faf17 100644 --- a/packages/nc-gui/lang/fa.json +++ b/packages/nc-gui/lang/fa.json @@ -171,7 +171,8 @@ "headLogin": "ورود | NocoDB", "resetPassword": "بازنشانی کلمه عبور شما", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "اعلان از طریق", diff --git a/packages/nc-gui/lang/fi.json b/packages/nc-gui/lang/fi.json index 7fe6c52b44..035045e498 100644 --- a/packages/nc-gui/lang/fi.json +++ b/packages/nc-gui/lang/fi.json @@ -171,7 +171,8 @@ "headLogin": "Kirjaudu sisään | Nokodb", "resetPassword": "Nollaa salasana", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Ilmoittaa kautta", diff --git a/packages/nc-gui/lang/fr.json b/packages/nc-gui/lang/fr.json index 62bc6df8a3..8c26789723 100644 --- a/packages/nc-gui/lang/fr.json +++ b/packages/nc-gui/lang/fr.json @@ -171,7 +171,8 @@ "headLogin": "Connexion | Nocodb", "resetPassword": "Réinitialiser le mot de passe", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Notifier via", diff --git a/packages/nc-gui/lang/hr.json b/packages/nc-gui/lang/hr.json index d5620b25c0..5be380f930 100644 --- a/packages/nc-gui/lang/hr.json +++ b/packages/nc-gui/lang/hr.json @@ -171,7 +171,8 @@ "headLogin": "Prijavite se | Nocodb", "resetPassword": "Vraćanje izvorne lozinke", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Obavijestiti putem", diff --git a/packages/nc-gui/lang/id.json b/packages/nc-gui/lang/id.json index 83537d209f..4704e039ca 100644 --- a/packages/nc-gui/lang/id.json +++ b/packages/nc-gui/lang/id.json @@ -171,7 +171,8 @@ "headLogin": "Masuk | Nocodb.", "resetPassword": "Mereset password Anda", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Beri tahu VIA.", diff --git a/packages/nc-gui/lang/it_IT.json b/packages/nc-gui/lang/it_IT.json index ec43960131..0f3bedce8d 100644 --- a/packages/nc-gui/lang/it_IT.json +++ b/packages/nc-gui/lang/it_IT.json @@ -171,7 +171,8 @@ "headLogin": "Accedi | NocoDB", "resetPassword": "Reimposta la tua password", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Notifica via.", diff --git a/packages/nc-gui/lang/iw.json b/packages/nc-gui/lang/iw.json index 7a79e9a388..45e373e57d 100644 --- a/packages/nc-gui/lang/iw.json +++ b/packages/nc-gui/lang/iw.json @@ -171,7 +171,8 @@ "headLogin": "התחבר | נוקודב", "resetPassword": "לאפס את הסיסמה שלך", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "תודיע דרך", diff --git a/packages/nc-gui/lang/ja.json b/packages/nc-gui/lang/ja.json index 6464c34f3a..40e5792eb1 100644 --- a/packages/nc-gui/lang/ja.json +++ b/packages/nc-gui/lang/ja.json @@ -171,7 +171,8 @@ "headLogin": "ログイン | NocoDB", "resetPassword": "パスワードをリセットする", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "次で通知する", diff --git a/packages/nc-gui/lang/ko.json b/packages/nc-gui/lang/ko.json index 2592a2aa88..a0b3876ad9 100644 --- a/packages/nc-gui/lang/ko.json +++ b/packages/nc-gui/lang/ko.json @@ -171,7 +171,8 @@ "headLogin": "로그인 | NocoDB.", "resetPassword": "비밀번호를 재설정", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "통지를 통지합니다", diff --git a/packages/nc-gui/lang/lv.json b/packages/nc-gui/lang/lv.json index 4f0d34f826..f2163b25e8 100644 --- a/packages/nc-gui/lang/lv.json +++ b/packages/nc-gui/lang/lv.json @@ -171,7 +171,8 @@ "headLogin": "Pieslēgties | NocoDB", "resetPassword": "Atjaunot paroli", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Paziņot izmantojot", diff --git a/packages/nc-gui/lang/nl.json b/packages/nc-gui/lang/nl.json index 5991b44b03..1e643d36b8 100644 --- a/packages/nc-gui/lang/nl.json +++ b/packages/nc-gui/lang/nl.json @@ -171,7 +171,8 @@ "headLogin": "Log in | Nocodb", "resetPassword": "Stel je wachtwoord opnieuw in", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Melding via", diff --git a/packages/nc-gui/lang/no.json b/packages/nc-gui/lang/no.json index bc1af25755..3714b6dddf 100644 --- a/packages/nc-gui/lang/no.json +++ b/packages/nc-gui/lang/no.json @@ -171,7 +171,8 @@ "headLogin": "Logg inn | NocoDB.", "resetPassword": "Tilbakestill passordet ditt", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Varsle Via.", diff --git a/packages/nc-gui/lang/pl.json b/packages/nc-gui/lang/pl.json index 1484c65482..051d85875e 100644 --- a/packages/nc-gui/lang/pl.json +++ b/packages/nc-gui/lang/pl.json @@ -171,7 +171,8 @@ "headLogin": "Zaloguj się | NOCODB.", "resetPassword": "Zresetuj swoje hasło", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Powiadomić VIA.", diff --git a/packages/nc-gui/lang/pt.json b/packages/nc-gui/lang/pt.json index d5f1cc0914..8961d7bdd6 100644 --- a/packages/nc-gui/lang/pt.json +++ b/packages/nc-gui/lang/pt.json @@ -171,7 +171,8 @@ "headLogin": "Autentique-se | Noco", "resetPassword": "Redefina a sua palavra-passe", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Notificar via.", diff --git a/packages/nc-gui/lang/pt_BR.json b/packages/nc-gui/lang/pt_BR.json index 5085da5e01..cb24beebc0 100644 --- a/packages/nc-gui/lang/pt_BR.json +++ b/packages/nc-gui/lang/pt_BR.json @@ -171,7 +171,8 @@ "headLogin": "Autentique-se | NocoDB", "resetPassword": "Redefina a sua senha", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Notificar via.", diff --git a/packages/nc-gui/lang/ru.json b/packages/nc-gui/lang/ru.json index 63dc7e7bbb..5ab0ef2225 100644 --- a/packages/nc-gui/lang/ru.json +++ b/packages/nc-gui/lang/ru.json @@ -171,7 +171,8 @@ "headLogin": "Войти |. NOCODB", "resetPassword": "Сбросить пароль", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Уведомлять через", diff --git a/packages/nc-gui/lang/sl.json b/packages/nc-gui/lang/sl.json index 6c397afcce..9f56795611 100644 --- a/packages/nc-gui/lang/sl.json +++ b/packages/nc-gui/lang/sl.json @@ -171,7 +171,8 @@ "headLogin": "Prijava | Nocodb.", "resetPassword": "Ponastavi geslo", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Obvestite VIO.", diff --git a/packages/nc-gui/lang/sv.json b/packages/nc-gui/lang/sv.json index df8914ae8c..de5eede18e 100644 --- a/packages/nc-gui/lang/sv.json +++ b/packages/nc-gui/lang/sv.json @@ -171,7 +171,8 @@ "headLogin": "Logga in | Nocodb", "resetPassword": "Återställ ditt lösenord", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Meddela via", diff --git a/packages/nc-gui/lang/th.json b/packages/nc-gui/lang/th.json index 286857aea7..0a8cf296bb 100644 --- a/packages/nc-gui/lang/th.json +++ b/packages/nc-gui/lang/th.json @@ -171,7 +171,8 @@ "headLogin": "เข้าสู่ระบบ nocodb", "resetPassword": "รีเซ็ตรหัสผ่านของคุณ", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "แจ้งเตือนผ่าน", diff --git a/packages/nc-gui/lang/tr.json b/packages/nc-gui/lang/tr.json index 8de333bfc0..79bbbe78d9 100644 --- a/packages/nc-gui/lang/tr.json +++ b/packages/nc-gui/lang/tr.json @@ -171,7 +171,8 @@ "headLogin": "Giriş | Nocodb", "resetPassword": "Şifrenizi sıfırlayın", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Aracılığıyla Bildir", diff --git a/packages/nc-gui/lang/uk.json b/packages/nc-gui/lang/uk.json index c9741ed867..1446b367d0 100644 --- a/packages/nc-gui/lang/uk.json +++ b/packages/nc-gui/lang/uk.json @@ -171,7 +171,8 @@ "headLogin": "Вхід | Нокодб", "resetPassword": "Скинути пароль", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Повідомити через", diff --git a/packages/nc-gui/lang/vi.json b/packages/nc-gui/lang/vi.json index c1569b0413..cfe69a98b9 100644 --- a/packages/nc-gui/lang/vi.json +++ b/packages/nc-gui/lang/vi.json @@ -171,7 +171,8 @@ "headLogin": "Đăng nhập |. NOCODB.", "resetPassword": "Đặt lại mật khẩu của bạn", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "Thông báo qua", diff --git a/packages/nc-gui/lang/zh_CN.json b/packages/nc-gui/lang/zh_CN.json index 5fa8b7e6c5..a39e3e8eab 100644 --- a/packages/nc-gui/lang/zh_CN.json +++ b/packages/nc-gui/lang/zh_CN.json @@ -171,7 +171,8 @@ "headLogin": "登录 | NocoDB", "resetPassword": "重置密码", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "通知Via", diff --git a/packages/nc-gui/lang/zh_HK.json b/packages/nc-gui/lang/zh_HK.json index 5d8e7a8721..614c1babeb 100644 --- a/packages/nc-gui/lang/zh_HK.json +++ b/packages/nc-gui/lang/zh_HK.json @@ -171,7 +171,8 @@ "headLogin": "登入 | NocoDB", "resetPassword": "reset你嗰密碼", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "通知Via", diff --git a/packages/nc-gui/lang/zh_TW.json b/packages/nc-gui/lang/zh_TW.json index 0150a809a8..6396f248a9 100644 --- a/packages/nc-gui/lang/zh_TW.json +++ b/packages/nc-gui/lang/zh_TW.json @@ -171,7 +171,8 @@ "headLogin": "登入｜NocoDB", "resetPassword": "重設密碼", "teamAndSettings": "Team & Settings", - "apiDocs": "API Docs" + "apiDocs": "API Docs", + "importFromAirtable": "Import From Airtable" }, "labels": { "notifyVia": "通知Via", diff --git a/packages/nc-gui/store/tabs.js b/packages/nc-gui/store/tabs.js index 0be2b5bbaf..3d0ac62b4a 100644 --- a/packages/nc-gui/store/tabs.js +++ b/packages/nc-gui/store/tabs.js @@ -290,6 +290,20 @@ export const actions = { } commit('list', tabs) }, + async loadFirstTableTab({ commit, state, rootGetters, dispatch, rootState }, load) { + const tabs = [] + + const nodes = rootState.project + .list[0] // project + .children[0] // environment + .children[0] // db + .children.find(n => n.type === 'tableDir') // parent node + .children + if (nodes && nodes[0]) { + tabs.push(nodes[0]) + } + if (tabs.length) { commit('list', tabs) } + }, removeTableTab({ commit, state }, nodes) { const tabs = JSON.parse(JSON.stringify(state.list)) diff --git a/packages/nocodb/package-lock.json b/packages/nocodb/package-lock.json index 3ed1843470..9b7f90dce5 100644 --- a/packages/nocodb/package-lock.json +++ b/packages/nocodb/package-lock.json @@ -12,6 +12,7 @@ "@google-cloud/storage": "^5.7.2", "@graphql-tools/merge": "^6.0.12", "@sentry/node": "^6.3.5", + "airtable": "^0.11.3", "archiver": "^5.0.2", "auto-bind": "^4.0.0", "aws-sdk": "^2.829.0", @@ -19,6 +20,7 @@ "bcryptjs": "^2.4.3", "body-parser": "^1.19.0", "boxen": "^5.0.0", + "bullmq": "^1.81.1", "clear": "^0.1.0", "cli-table3": "^0.6.0", "cluster": "^0.7.7", @@ -74,6 +76,7 @@ "ncp": "^2.0.0", "nocodb-sdk": "file:../nocodb-sdk", "nodemailer": "^6.4.10", + "object-hash": "^3.0.0", "ora": "^4.0.4", "os-locale": "^5.0.0", "papaparse": "^5.3.1", @@ -2367,6 +2370,11 @@ "node": ">=6.5" } }, + "node_modules/abortcontroller-polyfill": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.3.tgz", + "integrity": "sha512-zetDJxd89y3X99Kvo4qFx8GKlt6GsvN3UcRZHwU6iFA/0KiOmhkTVhe8oRoTBiTVPZu09x3vCra47+w8Yz1+2Q==" + }, "node_modules/accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -2498,6 +2506,26 @@ "node": ">=8" } }, + "node_modules/airtable": { + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/airtable/-/airtable-0.11.3.tgz", + "integrity": "sha512-fPT0SdipmJU1eQNe+sJshvnb87HcZ2rPS7gJuryDAF9xDbfbxB8AmYQSuC1f0PIbLyPPpSWBuEewf4UJ9i3rJw==", + "dependencies": { + "@types/node": ">=8.0.0 <15", + "abort-controller": "^3.0.0", + "abortcontroller-polyfill": "^1.4.0", + "lodash": "^4.17.21", + "node-fetch": "^2.6.7" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/airtable/node_modules/@types/node": { + "version": "14.18.16", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.16.tgz", + "integrity": "sha512-X3bUMdK/VmvrWdoTkz+VCn6nwKwrKCFTHtqwBIaQJNx4RUIBBUFXM00bqPz/DsDd+Icjmzm6/tyYZzeGVqb6/Q==" + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -4257,6 +4285,27 @@ "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=", "dev": true }, + "node_modules/bullmq": { + "version": "1.81.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.81.4.tgz", + "integrity": "sha512-sUEWOMKZnWlh1/XNqYAoSwXW6P8nZN7uJiHKZ8XlZCiIxWlEGjFtlugkkiCZ0lsTI2nNRHdxfpn78x9K3L1utQ==", + "dependencies": { + "cron-parser": "^4.2.1", + "get-port": "^5.1.1", + "glob": "^7.2.0", + "ioredis": "^4.28.5", + "lodash": "^4.17.21", + "msgpackr": "^1.4.6", + "semver": "^6.3.0", + "tslib": "^1.14.1", + "uuid": "^8.3.2" + } + }, + "node_modules/bullmq/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "node_modules/busboy": { "version": "0.2.14", "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", @@ -7324,6 +7373,17 @@ "moment-timezone": "^0.5.x" } }, + "node_modules/cron-parser": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.4.0.tgz", + "integrity": "sha512-TrE5Un4rtJaKgmzPewh67yrER5uKM0qI9hGLDBfWb8GGRe9pn/SDkhVrdHa4z7h0SeyeNxnQnogws/H+AQANQA==", + "dependencies": { + "luxon": "^1.28.0" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/cross-argv": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/cross-argv/-/cross-argv-1.0.0.tgz", @@ -10760,7 +10820,6 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", - "dev": true, "engines": { "node": ">=8" }, @@ -14520,6 +14579,14 @@ "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==", "dev": true }, + "node_modules/luxon": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", + "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==", + "engines": { + "node": "*" + } + }, "node_modules/mailersend": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/mailersend/-/mailersend-1.3.1.tgz", @@ -15728,6 +15795,104 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/msgpackr": { + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.5.7.tgz", + "integrity": "sha512-Hsa80i8W4BiObSMHslfnwC+CC1CYHZzoXJZn0+3EvoCEOgt3c5QlXhdcjgFk2aZxMgpV8aUFZqJyQUCIp4UrzA==", + "optionalDependencies": { + "msgpackr-extract": "^1.1.4" + } + }, + "node_modules/msgpackr-extract": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-1.1.4.tgz", + "integrity": "sha512-WQbHvsThprXh+EqZYy+SQFEs7z6bNM7a0vgirwUfwUcphWGT2mdPcpyLCNiRsN6w5q5VKJUMblHY+tNEyceb9Q==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "^4.3.2" + }, + "optionalDependencies": { + "msgpackr-extract-darwin-arm64": "1.1.0", + "msgpackr-extract-darwin-x64": "1.1.0", + "msgpackr-extract-linux-arm": "1.1.0", + "msgpackr-extract-linux-arm64": "1.1.0", + "msgpackr-extract-linux-x64": "1.1.0", + "msgpackr-extract-win32-x64": "1.1.0" + } + }, + "node_modules/msgpackr-extract-darwin-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-1.1.0.tgz", + "integrity": "sha512-s1kHoT12tS2cCQOv+Wl3I+/cYNJXBPtwQqGA+dPYoXmchhXiE0Nso+BIfvQ5PxbmAyjj54Q5o7PnLTqVquNfZA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/msgpackr-extract-darwin-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-1.1.0.tgz", + "integrity": "sha512-yx/H/i12IKg4eWGu/eKdKzJD4jaYvvujQSaVmeOMCesbSQnWo5X6YR9TFjoiNoU9Aexk1KufzL9gW+1DozG1yw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/msgpackr-extract-linux-arm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-1.1.0.tgz", + "integrity": "sha512-0VvSCqi12xpavxl14gMrauwIzHqHbmSChUijy/uo3mpjB1Pk4vlisKpZsaOZvNJyNKj0ACi5jYtbWnnOd7hYGw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/msgpackr-extract-linux-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-1.1.0.tgz", + "integrity": "sha512-AxFle3fHNwz2V4CYDIGFxI6o/ZuI0lBKg0uHI8EcCMUmDE5mVAUWYge5WXmORVvb8sVWyVgFlmi3MTu4Ve6tNQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/msgpackr-extract-linux-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-1.1.0.tgz", + "integrity": "sha512-O+XoyNFWpdB8oQL6O/YyzffPpmG5rTNrr1nKLW70HD2ENJUhcITzbV7eZimHPzkn8LAGls1tBaMTHQezTBpFOw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/msgpackr-extract-win32-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-1.1.0.tgz", + "integrity": "sha512-6AJdM5rNsL4yrskRfhujVSPEd6IBpgvsnIT/TPowKNLQ62iIdryizPY2PJNFiW3AJcY249AHEiDBXS1cTDPxzA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/mssql": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/mssql/-/mssql-6.4.0.tgz", @@ -16150,14 +16315,22 @@ "integrity": "sha512-ASCL5U13as7HhOExbT6OlWJJUV/lLzL2voOSP1UVehpRD8FbSrSDjfScK/KwAvVTI5AS6r4VwbOMlIqtvRidnA==" }, "node_modules/node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "dependencies": { "whatwg-url": "^5.0.0" }, "engines": { "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/node-forge": { @@ -16194,6 +16367,17 @@ "node": ">= 0.8.0" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-4.3.2.tgz", + "integrity": "sha512-P5Ep3ISdmwcCkZIaBaQamQtWAG0facC89phWZgi5Z3hBU//J6S48OIvyZWSPPf6yQMklLZiqoosWAZUj7N+esA==", + "optional": true, + "bin": { + "node-gyp-build-optional": "optional.js", + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-gyp/node_modules/nopt": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", @@ -17257,6 +17441,14 @@ "node": ">=0.10.0" } }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "engines": { + "node": ">= 6" + } + }, "node_modules/object-inspect": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz", @@ -26673,6 +26865,11 @@ "event-target-shim": "^5.0.0" } }, + "abortcontroller-polyfill": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.3.tgz", + "integrity": "sha512-zetDJxd89y3X99Kvo4qFx8GKlt6GsvN3UcRZHwU6iFA/0KiOmhkTVhe8oRoTBiTVPZu09x3vCra47+w8Yz1+2Q==" + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -26781,6 +26978,25 @@ } } }, + "airtable": { + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/airtable/-/airtable-0.11.3.tgz", + "integrity": "sha512-fPT0SdipmJU1eQNe+sJshvnb87HcZ2rPS7gJuryDAF9xDbfbxB8AmYQSuC1f0PIbLyPPpSWBuEewf4UJ9i3rJw==", + "requires": { + "@types/node": ">=8.0.0 <15", + "abort-controller": "^3.0.0", + "abortcontroller-polyfill": "^1.4.0", + "lodash": "^4.17.21", + "node-fetch": "^2.6.7" + }, + "dependencies": { + "@types/node": { + "version": "14.18.16", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.16.tgz", + "integrity": "sha512-X3bUMdK/VmvrWdoTkz+VCn6nwKwrKCFTHtqwBIaQJNx4RUIBBUFXM00bqPz/DsDd+Icjmzm6/tyYZzeGVqb6/Q==" + } + } + }, "ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -28213,6 +28429,29 @@ "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=", "dev": true }, + "bullmq": { + "version": "1.81.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.81.4.tgz", + "integrity": "sha512-sUEWOMKZnWlh1/XNqYAoSwXW6P8nZN7uJiHKZ8XlZCiIxWlEGjFtlugkkiCZ0lsTI2nNRHdxfpn78x9K3L1utQ==", + "requires": { + "cron-parser": "^4.2.1", + "get-port": "^5.1.1", + "glob": "^7.2.0", + "ioredis": "^4.28.5", + "lodash": "^4.17.21", + "msgpackr": "^1.4.6", + "semver": "^6.3.0", + "tslib": "^1.14.1", + "uuid": "^8.3.2" + }, + "dependencies": { + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + } + } + }, "busboy": { "version": "0.2.14", "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", @@ -30607,6 +30846,14 @@ "moment-timezone": "^0.5.x" } }, + "cron-parser": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.4.0.tgz", + "integrity": "sha512-TrE5Un4rtJaKgmzPewh67yrER5uKM0qI9hGLDBfWb8GGRe9pn/SDkhVrdHa4z7h0SeyeNxnQnogws/H+AQANQA==", + "requires": { + "luxon": "^1.28.0" + } + }, "cross-argv": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/cross-argv/-/cross-argv-1.0.0.tgz", @@ -33382,8 +33629,7 @@ "get-port": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", - "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", - "dev": true + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==" }, "get-stdin": { "version": "6.0.0", @@ -36211,6 +36457,11 @@ "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==", "dev": true }, + "luxon": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", + "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==" + }, "mailersend": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/mailersend/-/mailersend-1.3.1.tgz", @@ -37175,6 +37426,65 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "msgpackr": { + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.5.7.tgz", + "integrity": "sha512-Hsa80i8W4BiObSMHslfnwC+CC1CYHZzoXJZn0+3EvoCEOgt3c5QlXhdcjgFk2aZxMgpV8aUFZqJyQUCIp4UrzA==", + "requires": { + "msgpackr-extract": "^1.1.4" + } + }, + "msgpackr-extract": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-1.1.4.tgz", + "integrity": "sha512-WQbHvsThprXh+EqZYy+SQFEs7z6bNM7a0vgirwUfwUcphWGT2mdPcpyLCNiRsN6w5q5VKJUMblHY+tNEyceb9Q==", + "optional": true, + "requires": { + "msgpackr-extract-darwin-arm64": "1.1.0", + "msgpackr-extract-darwin-x64": "1.1.0", + "msgpackr-extract-linux-arm": "1.1.0", + "msgpackr-extract-linux-arm64": "1.1.0", + "msgpackr-extract-linux-x64": "1.1.0", + "msgpackr-extract-win32-x64": "1.1.0", + "node-gyp-build-optional-packages": "^4.3.2" + } + }, + "msgpackr-extract-darwin-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-1.1.0.tgz", + "integrity": "sha512-s1kHoT12tS2cCQOv+Wl3I+/cYNJXBPtwQqGA+dPYoXmchhXiE0Nso+BIfvQ5PxbmAyjj54Q5o7PnLTqVquNfZA==", + "optional": true + }, + "msgpackr-extract-darwin-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-1.1.0.tgz", + "integrity": "sha512-yx/H/i12IKg4eWGu/eKdKzJD4jaYvvujQSaVmeOMCesbSQnWo5X6YR9TFjoiNoU9Aexk1KufzL9gW+1DozG1yw==", + "optional": true + }, + "msgpackr-extract-linux-arm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-1.1.0.tgz", + "integrity": "sha512-0VvSCqi12xpavxl14gMrauwIzHqHbmSChUijy/uo3mpjB1Pk4vlisKpZsaOZvNJyNKj0ACi5jYtbWnnOd7hYGw==", + "optional": true + }, + "msgpackr-extract-linux-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-1.1.0.tgz", + "integrity": "sha512-AxFle3fHNwz2V4CYDIGFxI6o/ZuI0lBKg0uHI8EcCMUmDE5mVAUWYge5WXmORVvb8sVWyVgFlmi3MTu4Ve6tNQ==", + "optional": true + }, + "msgpackr-extract-linux-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-1.1.0.tgz", + "integrity": "sha512-O+XoyNFWpdB8oQL6O/YyzffPpmG5rTNrr1nKLW70HD2ENJUhcITzbV7eZimHPzkn8LAGls1tBaMTHQezTBpFOw==", + "optional": true + }, + "msgpackr-extract-win32-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-1.1.0.tgz", + "integrity": "sha512-6AJdM5rNsL4yrskRfhujVSPEd6IBpgvsnIT/TPowKNLQ62iIdryizPY2PJNFiW3AJcY249AHEiDBXS1cTDPxzA==", + "optional": true + }, "mssql": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/mssql/-/mssql-6.4.0.tgz", @@ -37556,9 +37866,9 @@ "integrity": "sha512-ASCL5U13as7HhOExbT6OlWJJUV/lLzL2voOSP1UVehpRD8FbSrSDjfScK/KwAvVTI5AS6r4VwbOMlIqtvRidnA==" }, "node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "requires": { "whatwg-url": "^5.0.0" } @@ -37605,6 +37915,12 @@ } } }, + "node-gyp-build-optional-packages": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-4.3.2.tgz", + "integrity": "sha512-P5Ep3ISdmwcCkZIaBaQamQtWAG0facC89phWZgi5Z3hBU//J6S48OIvyZWSPPf6yQMklLZiqoosWAZUj7N+esA==", + "optional": true + }, "node-libs-browser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", @@ -38454,6 +38770,11 @@ } } }, + "object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==" + }, "object-inspect": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz", diff --git a/packages/nocodb/package.json b/packages/nocodb/package.json index 522a9d2ff5..cc2cae2723 100644 --- a/packages/nocodb/package.json +++ b/packages/nocodb/package.json @@ -94,6 +94,7 @@ "@google-cloud/storage": "^5.7.2", "@graphql-tools/merge": "^6.0.12", "@sentry/node": "^6.3.5", + "airtable": "^0.11.3", "archiver": "^5.0.2", "auto-bind": "^4.0.0", "aws-sdk": "^2.829.0", @@ -101,6 +102,7 @@ "bcryptjs": "^2.4.3", "body-parser": "^1.19.0", "boxen": "^5.0.0", + "bullmq": "^1.81.1", "clear": "^0.1.0", "cli-table3": "^0.6.0", "cluster": "^0.7.7", @@ -156,6 +158,7 @@ "ncp": "^2.0.0", "nocodb-sdk": "file:../nocodb-sdk", "nodemailer": "^6.4.10", + "object-hash": "^3.0.0", "ora": "^4.0.4", "os-locale": "^5.0.0", "papaparse": "^5.3.1", diff --git a/packages/nocodb/src/lib/noco-jobs/EmitteryJobsMgr.ts b/packages/nocodb/src/lib/noco-jobs/EmitteryJobsMgr.ts new file mode 100644 index 0000000000..9b0291a566 --- /dev/null +++ b/packages/nocodb/src/lib/noco-jobs/EmitteryJobsMgr.ts @@ -0,0 +1,35 @@ +import JobsMgr from './JobsMgr'; +import Emittery from 'emittery'; + +export default class EmitteryJobsMgr extends JobsMgr { + emitter: Emittery; + + constructor() { + super(); + this.emitter = new Emittery(); + } + + add(jobName: string, payload: any): Promise { + return this.emitter.emit(jobName, payload); + } + + addJobWorker( + jobName: string, + workerFn: ( + payload: any, + progressCbk?: (payload: any, msg?: string) => void + ) => void + ) { + this.emitter.on(jobName, async payload => { + try { + await workerFn(payload, msg => + this.invokeProgressCbks(jobName, payload, msg) + ); + await this.invokeSuccessCbks(jobName, payload); + } catch (e) { + console.log(e); + await this.invokeFailureCbks(jobName, payload, e); + } + }); + } +} diff --git a/packages/nocodb/src/lib/noco-jobs/JobsMgr.ts b/packages/nocodb/src/lib/noco-jobs/JobsMgr.ts new file mode 100644 index 0000000000..13395ae5a4 --- /dev/null +++ b/packages/nocodb/src/lib/noco-jobs/JobsMgr.ts @@ -0,0 +1,65 @@ +export default abstract class JobsMgr { + protected successCbks: Array<{ + [jobName: string]: (payload: any) => void; + }> = []; + protected failureCbks: Array<{ + [jobName: string]: (payload: any, error: Error) => void; + }> = []; + protected progressCbks: Array<{ + [jobName: string]: (payload: any, msg?: string) => void; + }> = []; + + public abstract add(jobName: string, payload: T): Promise; + + public abstract addJobWorker( + jobName: string, + workerFn: ( + payload: any, + progressCbk?: (payload: any, msg?: string) => void + ) => void, + options?: { + onSuccess?: (payload: any) => void; + onFailure?: (payload: any, msg: string) => void; + onProgress?: (payload: any, msgOrData: any) => void; + } + ); + + addSuccessCbk(jobName: string, cbk: (payload: any) => void) { + this.successCbks[jobName] = this.successCbks[jobName] || []; + this.successCbks[jobName].push(cbk); + } + + addFailureCbk(jobName: string, cbk: (payload: any, msg: string) => void) { + this.failureCbks[jobName] = this.failureCbks[jobName] || []; + this.failureCbks[jobName].push(cbk); + } + addProgressCbk( + jobName: string, + cbk: (payload: any, progress: string) => void + ) { + this.progressCbks[jobName] = this.progressCbks[jobName] || []; + this.progressCbks[jobName].push(cbk); + } + + protected async invokeSuccessCbks(jobName: string, payload: any) { + await Promise.all(this.successCbks?.[jobName]?.map(cb => cb(payload))); + } + protected async invokeFailureCbks( + jobName: string, + payload: any, + error?: Error + ) { + await Promise.all( + this.failureCbks?.[jobName]?.map(cb => cb(payload, error)) + ); + } + protected async invokeProgressCbks( + jobName: string, + payload: any, + msg?: string + ) { + await Promise.all( + this.progressCbks?.[jobName]?.map(cb => cb(payload, msg)) + ); + } +} diff --git a/packages/nocodb/src/lib/noco-jobs/NocoJobs.ts b/packages/nocodb/src/lib/noco-jobs/NocoJobs.ts new file mode 100644 index 0000000000..dd19d5c1f9 --- /dev/null +++ b/packages/nocodb/src/lib/noco-jobs/NocoJobs.ts @@ -0,0 +1,20 @@ +import JobsMgr from './JobsMgr'; +import EmitteryJobsMgr from './EmitteryJobsMgr'; +import RedisJobsMgr from './RedisJobsMgr'; + +export default class NocoJobs { + private static client: JobsMgr; + + private static init() { + if (process.env.NC_REDIS_URL) { + this.client = new RedisJobsMgr(process.env.NC_REDIS_URL); + } else { + this.client = new EmitteryJobsMgr(); + } + } + + public static get jobsMgr(): JobsMgr { + if (!this.client) this.init(); + return this.client; + } +} diff --git a/packages/nocodb/src/lib/noco-jobs/RedisJobsMgr.ts b/packages/nocodb/src/lib/noco-jobs/RedisJobsMgr.ts new file mode 100644 index 0000000000..bc2551bbd9 --- /dev/null +++ b/packages/nocodb/src/lib/noco-jobs/RedisJobsMgr.ts @@ -0,0 +1,54 @@ +import { Queue, Worker } from 'bullmq'; +import Redis from 'ioredis'; +import JobsMgr from './JobsMgr'; + +export default class RedisJobsMgr extends JobsMgr { + queue: { [jobName: string]: Queue }; + workers: { [jobName: string]: Worker }; + connection: Redis; + + constructor(config: any) { + super(); + this.queue = {}; + this.workers = {}; + this.connection = new Redis(config); + } + + async add( + jobName: string, + payload: any + // options?: { + // onSuccess?: (payload: any) => void; + // onFailure?: (payload: any, msg: string) => void; + // onProgress?: (payload: any, msgOrData: any) => void; + // } + ): Promise { + this.queue[jobName] = + this.queue[jobName] || + new Queue(jobName, { connection: this.connection }); + this.queue[jobName].add(jobName, payload); + } + + addJobWorker( + jobName: string, + workerFn: ( + payload: any, + progressCbk?: (payload: any, msg?: string) => void + ) => void + ) { + this.workers[jobName] = new Worker( + jobName, + async payload => { + try { + await workerFn(payload.data, (...args) => + this.invokeProgressCbks(jobName, ...args) + ); + await this.invokeFailureCbks(jobName, payload.data); + } catch (e) { + await this.invokeFailureCbks(jobName, payload.data); + } + }, + { connection: this.connection } + ); + } +} diff --git a/packages/nocodb/src/lib/noco-models/Project.ts b/packages/nocodb/src/lib/noco-models/Project.ts index 25d8485e43..20009b3148 100644 --- a/packages/nocodb/src/lib/noco-models/Project.ts +++ b/packages/nocodb/src/lib/noco-models/Project.ts @@ -152,14 +152,14 @@ export default class Project implements ProjectType { deleted: false }); await NocoCache.set(`${CacheScope.PROJECT}:${projectId}`, projectData); - if (projectData.uuid) { + if (projectData?.uuid) { await NocoCache.set( `${CacheScope.PROJECT}:${projectData.uuid}`, projectId ); } } else { - if (projectData.deleted) { + if (projectData?.deleted) { projectData = null; } } diff --git a/packages/nocodb/src/lib/noco-models/SyncLogs.ts b/packages/nocodb/src/lib/noco-models/SyncLogs.ts new file mode 100644 index 0000000000..53a5b805bc --- /dev/null +++ b/packages/nocodb/src/lib/noco-models/SyncLogs.ts @@ -0,0 +1,57 @@ +import Noco from '../noco/Noco'; +import { MetaTable } from '../utils/globals'; + +export default class SyncLogs { + id?: string; + project_id?: string; + fk_sync_source_id?: string; + time_taken?: string; + status?: string; + status_details?: string; + + constructor(syncLog: Partial) { + Object.assign(this, syncLog); + } + + static async list(projectId: string, ncMeta = Noco.ncMeta) { + const syncLogs = await ncMeta.metaList(null, null, MetaTable.SYNC_LOGS, { + condition: { + project_id: projectId + }, + orderBy: { + created_at: 'asc' + } + }); + return syncLogs?.map(h => new SyncLogs(h)); + } + + public static async insert( + syncLog: Partial< + SyncLogs & { + created_at?; + updated_at?; + } + >, + ncMeta = Noco.ncMeta + ) { + const insertObj = { + project_id: syncLog?.project_id, + fk_sync_source_id: syncLog?.fk_sync_source_id, + time_taken: syncLog?.time_taken, + status: syncLog?.status, + status_details: syncLog?.status_details + }; + + const { id } = await ncMeta.metaInsert2( + null, + null, + MetaTable.SYNC_LOGS, + insertObj + ); + return new SyncLogs({ ...insertObj, id }); + } + + static async delete(syncLogId: any, ncMeta = Noco.ncMeta) { + return await ncMeta.metaDelete(null, null, MetaTable.SYNC_LOGS, syncLogId); + } +} diff --git a/packages/nocodb/src/lib/noco-models/SyncSource.ts b/packages/nocodb/src/lib/noco-models/SyncSource.ts new file mode 100644 index 0000000000..254abbc40e --- /dev/null +++ b/packages/nocodb/src/lib/noco-models/SyncSource.ts @@ -0,0 +1,135 @@ +import Noco from '../noco/Noco'; +import { MetaTable } from '../utils/globals'; +import extractProps from '../noco/meta/helpers/extractProps'; +import User from './User'; + +export default class SyncSource { + id?: string; + title?: string; + type?: string; + details?: any; + deleted?: boolean; + order?: number; + project_id?: string; + fk_user_id?: string; + + constructor(syncSource: Partial) { + Object.assign(this, syncSource); + } + + public getUser(ncMeta = Noco.ncMeta) { + return User.get(this.fk_user_id, ncMeta); + } + + public static async get(syncSourceId: string, ncMeta = Noco.ncMeta) { + const syncSource = await ncMeta.metaGet2( + null, + null, + MetaTable.SYNC_SOURCE, + syncSourceId + ); + if (syncSource.details && typeof syncSource.details === 'string') { + try { + syncSource.details = JSON.parse(syncSource.details); + } catch {} + } + return syncSource && new SyncSource(syncSource); + } + + static async list(projectId: string, ncMeta = Noco.ncMeta) { + const syncSources = await ncMeta.metaList( + null, + null, + MetaTable.SYNC_SOURCE, + { + condition: { + project_id: projectId + }, + orderBy: { + created_at: 'asc' + } + } + ); + + for (const syncSource of syncSources) { + if (syncSource.details && typeof syncSource.details === 'string') { + try { + syncSource.details = JSON.parse(syncSource.details); + } catch {} + } + } + return syncSources?.map(h => new SyncSource(h)); + } + + public static async insert( + syncSource: Partial< + SyncSource & { + created_at?; + updated_at?; + } + >, + ncMeta = Noco.ncMeta + ) { + const insertObj = { + id: syncSource?.id, + title: syncSource?.title, + type: syncSource?.type, + details: syncSource?.details, + project_id: syncSource?.project_id, + fk_user_id: syncSource?.fk_user_id + }; + + if (insertObj.details && typeof insertObj.details === 'object') { + insertObj.details = JSON.stringify(insertObj.details); + } + + const { id } = await ncMeta.metaInsert2( + null, + null, + MetaTable.SYNC_SOURCE, + insertObj + ); + + return this.get(id, ncMeta); + } + + public static async update( + syncSourceId: string, + syncSource: Partial, + ncMeta = Noco.ncMeta + ) { + const updateObj = extractProps(syncSource, [ + 'id', + 'title', + 'type', + 'details', + 'deleted', + 'order', + 'project_id' + ]); + + if (updateObj.details && typeof updateObj.details === 'object') { + updateObj.details = JSON.stringify(updateObj.details); + } + + // set meta + await ncMeta.metaUpdate( + null, + null, + MetaTable.SYNC_SOURCE, + updateObj, + syncSourceId + ); + + return this.get(syncSourceId, ncMeta); + } + + static async delete(syncSourceId: any, ncMeta = Noco.ncMeta) { + return await ncMeta.metaDelete( + null, + null, + MetaTable.SYNC_SOURCE, + syncSourceId + ); + } +} diff --git a/packages/nocodb/src/lib/noco/common/XcMigrationSourcev2.ts b/packages/nocodb/src/lib/noco/common/XcMigrationSourcev2.ts index 25df4729f9..02c182bdeb 100644 --- a/packages/nocodb/src/lib/noco/common/XcMigrationSourcev2.ts +++ b/packages/nocodb/src/lib/noco/common/XcMigrationSourcev2.ts @@ -1,5 +1,6 @@ import * as nc_011 from '../migrationsv2/nc_011'; import * as nc_012_alter_column_data_types from '../migrationsv2/nc_012_alter_column_data_types'; +import * as nc_013_sync_source from '../migrationsv2/nc_013_sync_source'; // Create a custom migration source class export default class XcMigrationSourcev2 { @@ -8,7 +9,11 @@ export default class XcMigrationSourcev2 { // arguments to getMigrationName and getMigration public getMigrations(): Promise { // In this example we are just returning migration names - return Promise.resolve(['nc_011', 'nc_012_alter_column_data_types']); + return Promise.resolve([ + 'nc_011', + 'nc_012_alter_column_data_types', + 'nc_013_sync_source' + ]); } public getMigrationName(migration): string { @@ -21,6 +26,8 @@ export default class XcMigrationSourcev2 { return nc_011; case 'nc_012_alter_column_data_types': return nc_012_alter_column_data_types; + case 'nc_013_sync_source': + return nc_013_sync_source; } } } diff --git a/packages/nocodb/src/lib/noco/meta/api/index.ts b/packages/nocodb/src/lib/noco/meta/api/index.ts index adf0ca248f..25f06395bf 100644 --- a/packages/nocodb/src/lib/noco/meta/api/index.ts +++ b/packages/nocodb/src/lib/noco/meta/api/index.ts @@ -41,11 +41,15 @@ import { publicMetaApis } from './publicApis'; import { Tele } from 'nc-help'; -import { Server } from 'socket.io'; +import { Server, Socket } from 'socket.io'; import passport from 'passport'; import crypto from 'crypto'; import swaggerApis from './swagger/swaggerApis'; +import importApis from './sync/importApis'; +import syncSourceApis from './sync/syncSourceApis'; + +const clients: { [id: string]: Socket } = {}; export default function(router: Router, server) { initStrategies(router); @@ -85,6 +89,7 @@ export default function(router: Router, server) { router.use(apiTokenApis); router.use(hookFilterApis); router.use(swaggerApis); + router.use(syncSourceApis); userApis(router); @@ -109,6 +114,7 @@ export default function(router: Router, server) { } )(socket.handshake, {}, next); }).on('connection', socket => { + clients[socket.id] = socket; const id = getHash( (process.env.NC_SERVER_UUID || Tele.id) + (socket?.handshake as any)?.user?.id @@ -121,6 +127,8 @@ export default function(router: Router, server) { Tele.event({ ...args, id }); }); }); + + importApis(router, clients); } function getHash(str) { diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/NocoSyncDestAdapter.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/NocoSyncDestAdapter.ts new file mode 100644 index 0000000000..8af4a493c5 --- /dev/null +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/NocoSyncDestAdapter.ts @@ -0,0 +1,6 @@ +export abstract class NocoSyncSourceAdapter { + public abstract init(): Promise; + public abstract destProjectWrite(): Promise; + public abstract destSchemaWrite(): Promise; + public abstract destDataWrite(): Promise; +} diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/NocoSyncSourceAdapter.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/NocoSyncSourceAdapter.ts new file mode 100644 index 0000000000..a419d1c24d --- /dev/null +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/NocoSyncSourceAdapter.ts @@ -0,0 +1,7 @@ +export abstract class NocoSyncSourceAdapter { + public abstract init(): Promise; + public abstract srcSchemaGet(): Promise; + public abstract srcDataLoad(): Promise; + public abstract srcDataListen(): Promise; + public abstract srcDataPoll(): Promise; +} diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/fetchAT.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/fetchAT.ts new file mode 100644 index 0000000000..4da2497800 --- /dev/null +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/fetchAT.ts @@ -0,0 +1,170 @@ +const axios = require('axios').default; + +var info : any = { + initialized: false +}; + +async function initialize(shareId) { + info.cookie = ""; + let url = `https://airtable.com/${shareId}` + + try { + const hreq = await axios.get(url, { + "headers": { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "accept-language": "en-US,en;q=0.9", + "sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Linux\"", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "none", + "sec-fetch-user": "?1", + "upgrade-insecure-requests": "1", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36" + }, + "referrerPolicy": "strict-origin-when-cross-origin", + "body": null, + "method": "GET" + }) + .then(response => { + for(const ck of response.headers['set-cookie']) { + info.cookie += ck.split(';')[0] + '; ' + } + return response.data + }) + + info.headers = JSON.parse(hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim()) + info.link = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim()) + info.baseInfo = decodeURIComponent(info.link).match(/{(.*)}/g)[0].split('&').reduce((result, el) => { + try { + return Object.assign(result, JSON.parse((el.includes('=')?el.split('=')[1]:el))) + } catch(e) { + if(el.includes('=')) { + return Object.assign(result, { [el.split('=')[0]]: el.split('=')[1]}) + } + } + }, {}) + info.baseId = info.baseInfo.applicationId; + info.initialized = true; + } catch (e) { + console.log(e); + info.initialized = false; + } +} + +async function read() { + if (info.initialized) { + const resreq = await axios("https://airtable.com" + info.link, { + "headers": { + "accept": "*/*", + "accept-language": "en-US,en;q=0.9", + "sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Linux\"", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-origin", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36", + "x-time-zone": "Europe/Berlin", + "cookie": info.cookie, + ...info.headers + }, + "referrerPolicy": "no-referrer", + "body": null, + "method": "GET" + }) + .then(response => { + return response.data + }) + .catch(() => { + throw "Error while fetching" + }) + + return {schema: resreq.data, baseId: info.baseId, baseInfo: info.baseInfo} + } else { + throw "Please initialize first!" + } +} + +async function readView(viewId) { + if (info.initialized) { + const resreq = await axios(`https://airtable.com/v0.3/view/${viewId}/readData?` + `stringifiedObjectParams=${ + encodeURIComponent('{}')}&requestId=${info.baseInfo.requestId}&accessPolicy=${encodeURIComponent(JSON.stringify({allowedActions: info.baseInfo.allowedActions, shareId: info.baseInfo.shareId, applicationId: info.baseInfo.applicationId, generationNumber: info.baseInfo.generationNumber, expires: info.baseInfo.expires, signature: info.baseInfo.signature}))}`, { + "headers": { + "accept": "*/*", + "accept-language": "en-US,en;q=0.9", + "sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Linux\"", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-origin", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36", + "x-time-zone": "Europe/Berlin", + "cookie": info.cookie, + ...info.headers + }, + "referrerPolicy": "no-referrer", + "body": null, + "method": "GET" + }) + .then(response => { + return response.data + }) + .catch(() => { + throw "Error while fetching" + }) + return {view: resreq.data} + } else { + throw "Please initialize first!" + } +} + +async function readTemplate(templateId) { + if (!info.initialized) { + await initialize('shrO8aYf3ybwSdDKn'); + } + const resreq = await axios(`https://www.airtable.com/v0.3/exploreApplications/${templateId}`, { + "headers": { + "accept": "*/*", + "accept-language": "en-US,en;q=0.9", + "sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Linux\"", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-origin", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36", + "x-time-zone": "Europe/Berlin", + "cookie": info.cookie, + ...info.headers + }, + "referrer": "https://www.airtable.com/", + "referrerPolicy": "same-origin", + "body": null, + "method": "GET", + "mode": "cors", + "credentials": "include" + }) + .then(response => { + return response.data + }) + .catch(() => { + throw "Error while fetching" + }) + return {template: resreq} +} + +function unicodeToChar(text) { + return text.replace(/\\u[\dA-F]{4}/gi, function(match) { + return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16)); + }); +} + +export default { + initialize, + read, + readView, + readTemplate +} \ No newline at end of file diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts new file mode 100644 index 0000000000..e9a8848bb9 --- /dev/null +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -0,0 +1,2007 @@ +import FetchAT from './fetchAT'; +import { UITypes } from 'nocodb-sdk'; +// import * as sMap from './syncMap'; +import FormData from 'form-data'; + +import { Api } from 'nocodb-sdk'; + +import axios from 'axios'; +import Airtable from 'airtable'; +import jsonfile from 'jsonfile'; +import hash from 'object-hash'; + +import dayjs from 'dayjs'; +import utc from 'dayjs/plugin/utc'; +dayjs.extend(utc); + +export default async ( + syncDB: AirtableSyncConfig, + progress: (msg: string) => void +) => { + const sMap = { + mapTbl: {}, + + // static mapping records between aTblId && ncId + addToMappingTbl(aTblId, ncId, ncName, parent?) { + this.mapTbl[aTblId] = { + ncId: ncId, + ncParent: parent, + // name added to assist in quick debug + ncName: ncName + }; + }, + + // get NcID from airtable ID + getNcIdFromAtId(aId) { + return this.mapTbl[aId]?.ncId; + }, + + // get nc Parent from airtable ID + getNcParentFromAtId(aId) { + return this.mapTbl[aId]?.ncParent; + }, + + // get nc-title from airtable ID + getNcNameFromAtId(aId) { + return this.mapTbl[aId]?.ncName; + } + }; + + let base, baseId; + const start = Date.now(); + const enableErrorLogs = false; + const process_aTblData = true; + const generate_migrationStats = true; + const debugMode = true; + let api: Api; + let g_aTblSchema = []; + let ncCreatedProjectSchema: any = {}; + const ncLinkMappingTable: any[] = []; + const aTblDataLinks: any[] = []; + const nestedLookupTbl: any[] = []; + const nestedRollupTbl: any[] = []; + const runTimeCounters = { + sort: 0, + filter: 0, + view: { + total: 0, + grid: 0, + gallery: 0, + form: 0 + }, + fetchAt: { + count: 0, + time: 0 + }, + columnNotMigrated: { + count: 0, + log: [] + } + }; + + function addToSkipColumnLog(tbl, col, type, reason?) { + runTimeCounters.columnNotMigrated.count++; + runTimeCounters.columnNotMigrated.log.push( + `tn[${tbl}] cn[${col}] type[${type}] :: ${reason}` + ); + } + + let syncLog = _log => { + // console.log(log) + }; + + // mapping table + // + + async function getAtableSchema(sDB) { + const start = Date.now(); + if(sDB.shareId.startsWith('exp')) { + const template = await FetchAT.readTemplate(sDB.shareId); + await FetchAT.initialize(template.template.exploreApplication.shareId); + } else { + await FetchAT.initialize(sDB.shareId); + } + const ft = await FetchAT.read(); + const duration = Date.now() - start; + runTimeCounters.fetchAt.count++; + runTimeCounters.fetchAt.time += duration; + + const file = ft.schema; + baseId = ft.baseId; + base = new Airtable({ apiKey: sDB.apiKey }).base(baseId); + // store copy of atbl schema globally + g_aTblSchema = file.tableSchemas; + + if (debugMode) jsonfile.writeFileSync('aTblSchema.json', ft, { spaces: 2 }); + + return file; + } + + async function getViewData(viewId) { + const start = Date.now(); + const ft = await FetchAT.readView(viewId); + const duration = Date.now() - start; + runTimeCounters.fetchAt.count++; + runTimeCounters.fetchAt.time += duration; + + if (debugMode) jsonfile.writeFileSync(`${viewId}.json`, ft, { spaces: 2 }); + return ft.view; + } + + // base mapping table + const aTblNcTypeMap = { + foreignKey: UITypes.LinkToAnotherRecord, + text: UITypes.SingleLineText, + multilineText: UITypes.LongText, + multipleAttachment: UITypes.Attachment, + checkbox: UITypes.Checkbox, + multiSelect: UITypes.MultiSelect, + select: UITypes.SingleSelect, + collaborator: UITypes.Collaborator, + multiCollaborator: UITypes.Collaborator, + date: UITypes.Date, + // kludge: phone: UITypes.PhoneNumber, + phone: UITypes.SingleLineText, + number: UITypes.Number, + rating: UITypes.Rating, + formula: UITypes.Formula, + rollup: UITypes.Rollup, + count: UITypes.Count, + lookup: UITypes.Lookup, + autoNumber: UITypes.AutoNumber, + barcode: UITypes.Barcode, + button: UITypes.Button + }; + + //----------------------------------------------------------------------------- + // aTbl helper routines + // + + function nc_sanitizeName(name) { + // knex complains use of '?' in field name + // good to replace all special characters by _ in one go + const col_name = name + .replace(/\?/g, 'QQ') + .replace('.', '_') + .trim(); + + return col_name; + } + + function nc_getSanitizedColumnName(table, name) { + const col_name = nc_sanitizeName(name); + + // check if already a column exists with same name? + const duplicateColumn = table.columns.find(x => x.title === name.trim()); + if (duplicateColumn) { + if (enableErrorLogs) console.log(`## Duplicate ${name.trim()}`); + } + + return { + // kludge: error observed in Nc with space around column-name + title: name.trim() + (duplicateColumn ? '_2' : ''), + column_name: col_name + (duplicateColumn ? '_2' : '') + }; + } + + // aTbl: retrieve table name from table ID + // + function aTbl_getTableName(tblId) { + const sheetObj = g_aTblSchema.find(tbl => tbl.id === tblId); + return { + tn: sheetObj.name + }; + } + + const ncSchema = { + tables: [], + tablesById: {} + }; + + // aTbl: retrieve column name from column ID + // + function aTbl_getColumnName(colId): any { + for (let i = 0; i < g_aTblSchema.length; i++) { + const sheetObj = g_aTblSchema[i]; + const column = sheetObj.columns.find(col => col.id === colId); + if (column !== undefined) + return { + tn: sheetObj.name, + cn: column.name + }; + } + } + + // nc dump schema + // + // @ts-ignore + async function nc_DumpTableSchema() { + console.log('['); + const ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); + for (let i = 0; i < ncTblList.list.length; i++) { + const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + console.log(JSON.stringify(ncTbl, null, 2)); + console.log(','); + } + console.log(']'); + } + + // retrieve nc column schema from using aTbl field ID as reference + // + async function nc_getColumnSchema(aTblFieldId) { + // let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); + // let aTblField = aTbl_getColumnName(aTblFieldId); + // let ncTblId = ncTblList.list.filter(x => x.title === aTblField.tn)[0].id; + // let ncTbl = await api.dbTable.read(ncTblId); + // let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn); + // return ncCol; + + const ncTblId = sMap.getNcParentFromAtId(aTblFieldId); + const ncColId = sMap.getNcIdFromAtId(aTblFieldId); + + // not migrated column, skip + if (ncColId === undefined || ncTblId === undefined) return 0; + + const ncCol = ncSchema.tablesById[ncTblId].columns.find( + x => x.id === ncColId + ); + return ncCol; + } + + // retrieve nc table schema using table name + // optimize: create a look-up table & re-use information + // + async function nc_getTableSchema(tableName) { + // let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); + // let ncTblId = ncTblList.list.filter(x => x.title === tableName)[0].id; + // let ncTbl = await api.dbTable.read(ncTblId); + // return ncTbl; + + return ncSchema.tables.find(x => x.title === tableName); + } + + // delete project if already exists + async function init({ + projectName + }: { + projectName?: string; + projectId?: string; + }) { + // delete 'sample' project if already exists + const x = await api.project.list(); + + const sampleProj = x.list.find(a => a.title === projectName); + if (sampleProj) { + await api.project.delete(sampleProj.id); + } + syncLog('Init'); + } + + // map UIDT + // + function getNocoType(col) { + // start with default map + let ncType = aTblNcTypeMap[col.type]; + + // types email & url are marked as text + // types currency & percent, duration are marked as number + // types createTime & modifiedTime are marked as formula + + switch (col.type) { + case 'text': + if (col.typeOptions?.validatorName === 'email') ncType = UITypes.Email; + else if (col.typeOptions?.validatorName === 'url') ncType = UITypes.URL; + break; + + case 'number': + // kludge: currency validation error with decimal places + if (col.typeOptions?.format === 'percentV2') ncType = UITypes.Percent; + else if (col.typeOptions?.format === 'duration') + ncType = UITypes.Duration; + else if (col.typeOptions?.format === 'currency') + ncType = UITypes.Currency; + break; + + case 'formula': + if (col.typeOptions?.formulaTextParsed === 'CREATED_TIME()') + ncType = UITypes.CreateTime; + else if (col.typeOptions?.formulaTextParsed === 'LAST_MODIFIED_TIME()') + ncType = UITypes.LastModifiedTime; + break; + + case 'computation': + if (col.typeOptions?.resultType === 'collaborator') + ncType = UITypes.Collaborator; + break; + + case 'date': + if (col.typeOptions?.isDateTime) ncType = UITypes.DateTime; + break; + + // case 'barcode': + // case 'button': + // ncType = UITypes.SingleLineText; + // break; + } + + return ncType; + } + + // retrieve additional options associated with selected data types + // + function getNocoTypeOptions(col: any): any { + switch (col.type) { + case 'select': + case 'multiSelect': { + // prepare options list in CSV format + // note: NC doesn't allow comma's in options + // + const opt = []; + for (const [, value] of Object.entries(col.typeOptions.choices)) { + opt.push((value as any).name); + sMap.addToMappingTbl( + (value as any).id, + undefined, + (value as any).name + ); + } + const csvOpt = "'" + opt.join("','") + "'"; + return { type: 'select', data: csvOpt }; + } + default: + return { type: undefined }; + } + } + + // convert to Nc schema (basic, excluding relations) + // + function tablesPrepare(tblSchema: any[]) { + const tables: any[] = []; + + for (let i = 0; i < tblSchema.length; ++i) { + const table: any = {}; + + syncLog(`Preparing base schema (sans relations): ${tblSchema[i].name}`); + runTimeCounters.view.total += tblSchema[i].views.length; + + // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id; + table.title = tblSchema[i].name; + table.table_name = nc_sanitizeName(tblSchema[i].name); + + // insert _aTbl_nc_rec_id of type ID by default + table.columns = [ + { + title: '_aTbl_nc_rec_id', + column_name: '_aTbl_nc_rec_id', + // uidt: UITypes.ID + uidt: UITypes.SingleLineText, + pk: true, + // mysql additionally requires NOT-NULL to be explicitly set + rqd: true + }, + { + title: '_aTbl_nc_rec_hash', + column_name: '_aTbl_nc_rec_hash', + uidt: UITypes.SingleLineText + } + ]; + + for (let j = 0; j < tblSchema[i].columns.length; j++) { + const col = tblSchema[i].columns[j]; + + // skip link, lookup, rollup fields in this iteration + if (['foreignKey', 'lookup', 'rollup'].includes(col.type)) { + continue; + } + + // base column schema + const ncName: any = nc_getSanitizedColumnName(table, col.name); + const ncCol: any = { + // Enable to use aTbl identifiers as is: id: col.id, + title: ncName.title, + column_name: ncName.column_name, + uidt: getNocoType(col) + }; + + // not supported datatype: pure formula field + // allow formula based computed fields (created time/ modified time to go through) + if (ncCol.uidt === UITypes.Formula) { + addToSkipColumnLog( + tblSchema[i].name, + ncName.title, + col.type, + 'column type not supported' + ); + continue; + } + + // populate cdf (column default value) if configured + if (col?.default) { + ncCol.cdf = col.default; + } + + // additional column parameters when applicable + const colOptions = getNocoTypeOptions(col); + + switch (colOptions.type) { + case 'select': + ncCol.dtxp = colOptions.data; + break; + + case undefined: + break; + } + table.columns.push(ncCol); + } + tables.push(table); + } + return tables; + } + + async function nocoCreateBaseSchema(aTblSchema) { + // base schema preparation: exclude + const tables: any[] = tablesPrepare(aTblSchema); + + syncLog(`Total tables: ${tables.length} `); + + // for each table schema, create nc table + for (let idx = 0; idx < tables.length; idx++) { + syncLog( + `[${idx + 1}/${tables.length}] Creating base table schema: ${ + tables[idx].title + }` + ); + + syncLog(`NC API: dbTable.create ${tables[idx].title}`); + const table: any = await api.dbTable.create( + ncCreatedProjectSchema.id, + tables[idx] + ); + updateNcTblSchema(table); + + // update mapping table + await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title); + for (let colIdx = 0; colIdx < table.columns.length; colIdx++) { + const aId = aTblSchema[idx].columns.find( + x => x.name.trim() === table.columns[colIdx].title + )?.id; + if (aId) + await sMap.addToMappingTbl( + aId, + table.columns[colIdx].id, + table.columns[colIdx].title, + table.id + ); + } + + // update default view name- to match it to airtable view name + syncLog(`NC API: dbView.list ${table.id}`); + const view = await api.dbView.list(table.id); + + syncLog( + `NC API: dbView.update ${view.list[0].id} ${aTblSchema[idx].views[0].name}` + ); + const aTbl_grid = aTblSchema[idx].views.find(x => x.type === 'grid'); + // @ts-ignore + const x = await api.dbView.update(view.list[0].id, { + title: aTbl_grid.name + }); + await updateNcTblSchemaById(table.id); + + await sMap.addToMappingTbl( + aTbl_grid.id, + table.views[0].id, + aTbl_grid.name, + table.id + ); + } + + // debug + // console.log(JSON.stringify(tables, null, 2)); + return tables; + } + + async function nocoCreateLinkToAnotherRecord(aTblSchema) { + // Link to another RECORD + for (let idx = 0; idx < aTblSchema.length; idx++) { + const aTblLinkColumns = aTblSchema[idx].columns.filter( + x => x.type === 'foreignKey' + ); + + // Link columns exist + // + if (aTblLinkColumns.length) { + for (let i = 0; i < aTblLinkColumns.length; i++) { + syncLog( + `[${idx + 1}/${aTblSchema.length}] Configuring Links :: [${i + 1}/${ + aTblLinkColumns.length + }] ${aTblSchema[idx].name}` + ); + + // for self links, there is no symmetric column + { + const src = aTbl_getColumnName(aTblLinkColumns[i].id); + const dst = aTbl_getColumnName( + aTblLinkColumns[i].typeOptions?.symmetricColumnId + ); + syncLog( + ` LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst?.tn}:${dst?.cn}` + ); + } + + // check if link already established? + if (!nc_isLinkExists(aTblLinkColumns[i].id)) { + // parent table ID + // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + const srcTableId = sMap.getNcIdFromAtId(aTblSchema[idx].id); + + // find child table name from symmetric column ID specified + // self link, symmetricColumnId field will be undefined + const childTable = aTbl_getColumnName( + aTblLinkColumns[i].typeOptions?.symmetricColumnId + ); + + // retrieve child table ID (nc) from table name + let childTableId = srcTableId; + if (childTable) { + childTableId = (await nc_getTableSchema(childTable.tn)).id; + } + + // check if already a column exists with this name? + const srcTbl: any = await api.dbTable.read(srcTableId); + + // create link + const ncName = nc_getSanitizedColumnName( + srcTbl, + aTblLinkColumns[i].name + ); + const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + uidt: UITypes.LinkToAnotherRecord, + title: ncName.title, + column_name: ncName.column_name, + parentId: srcTableId, + childId: childTableId, + type: 'mm' + // aTblLinkColumns[i].typeOptions.relationship === 'many' + // ? 'mm' + // : 'hm' + }); + updateNcTblSchema(ncTbl); + syncLog(`NC API: dbTableColumn.create LinkToAnotherRecord`); + + const ncId = ncTbl.columns.find(x => x.title === ncName.title)?.id; + await sMap.addToMappingTbl( + aTblLinkColumns[i].id, + ncId, + ncName.title, + ncTbl.id + ); + + // store link information in separate table + // this information will be helpful in identifying relation pair + const link = { + nc: { + title: aTblLinkColumns[i].name, + parentId: srcTableId, + childId: childTableId, + type: 'mm' + }, + aTbl: { + tblId: aTblSchema[idx].id, + ...aTblLinkColumns[i] + } + }; + + ncLinkMappingTable.push(link); + } else { + // if link already exists, we need to change name of linked column + // to what is represented in airtable + + // 1. extract associated link information from link table + // 2. retrieve parent table information (source) + // 3. using foreign parent & child column ID, find associated mapping in child table + // 4. update column name + const x = ncLinkMappingTable.findIndex( + x => + x.aTbl.tblId === + aTblLinkColumns[i].typeOptions.foreignTableId && + x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId + ); + + const childTblSchema: any = await api.dbTable.read( + ncLinkMappingTable[x].nc.childId + ); + const parentTblSchema: any = await api.dbTable.read( + ncLinkMappingTable[x].nc.parentId + ); + + // fix me + // let childTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.childId] + // let parentTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.parentId] + + let parentLinkColumn = parentTblSchema.columns.find( + col => col.title === ncLinkMappingTable[x].nc.title + ); + + // hack // fix me + if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') { + parentLinkColumn = parentTblSchema.columns.find( + col => col.title === ncLinkMappingTable[x].nc.title + '_2' + ); + } + + let childLinkColumn: any = {}; + + if (parentLinkColumn.colOptions.type == 'hm') { + // for hm: + // mapping between child & parent column id is direct + // + childLinkColumn = childTblSchema.columns.find( + col => + col.uidt === UITypes.LinkToAnotherRecord && + col.colOptions.fk_child_column_id === + parentLinkColumn.colOptions.fk_child_column_id && + col.colOptions.fk_parent_column_id === + parentLinkColumn.colOptions.fk_parent_column_id + ); + } else { + // for mm: + // mapping between child & parent column id is inverted + // + childLinkColumn = childTblSchema.columns.find( + col => + col.uidt === UITypes.LinkToAnotherRecord && + col.colOptions.fk_child_column_id === + parentLinkColumn.colOptions.fk_parent_column_id && + col.colOptions.fk_parent_column_id === + parentLinkColumn.colOptions.fk_child_column_id && + col.colOptions.fk_mm_model_id === + parentLinkColumn.colOptions.fk_mm_model_id + ); + } + + // check if already a column exists with this name? + const duplicate = childTblSchema.columns.find( + x => x.title === aTblLinkColumns[i].name + ); + const suffix = duplicate ? '_2' : ''; + if (duplicate) + if (enableErrorLogs) + console.log(`## Duplicate ${aTblLinkColumns[i].name}`); + + // rename + // note that: current rename API requires us to send all parameters, + // not just title being renamed + const ncName = nc_getSanitizedColumnName( + childTblSchema, + aTblLinkColumns[i].name + ); + const ncTbl: any = await api.dbTableColumn.update( + childLinkColumn.id, + { + ...childLinkColumn, + title: ncName.title, + column_name: ncName.column_name + } + ); + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + x => x.title === aTblLinkColumns[i].name + suffix + )?.id; + await sMap.addToMappingTbl( + aTblLinkColumns[i].id, + ncId, + aTblLinkColumns[i].name + suffix, + ncTbl.id + ); + + // console.log(res.columns.find(x => x.title === aTblLinkColumns[i].name)) + syncLog(`NC API: dbTableColumn.update rename symmetric column`); + } + } + } + } + } + + async function nocoCreateLookups(aTblSchema) { + // LookUps + for (let idx = 0; idx < aTblSchema.length; idx++) { + const aTblColumns = aTblSchema[idx].columns.filter( + x => x.type === 'lookup' + ); + + // parent table ID + // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + const srcTableId = sMap.getNcIdFromAtId(aTblSchema[idx].id); + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + if (aTblColumns.length) { + // Lookup + for (let i = 0; i < aTblColumns.length; i++) { + syncLog( + `[${idx + 1}/${aTblSchema.length}] Configuring Lookup :: [${i + + 1}/${aTblColumns.length}] ${aTblSchema[idx].name}` + ); + + // something is not right, skip + if ( + aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + if (enableErrorLogs) + console.log(`## Invalid column IDs mapped; skip`); + + addToSkipColumnLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + 'invalid column ID in dependency list' + ); + continue; + } + + const ncRelationColumnId = sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.relationColumnId + ); + const ncLookupColumnId = sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + + if (ncLookupColumnId === undefined) { + aTblColumns[i]['srcTableId'] = srcTableId; + nestedLookupTbl.push(aTblColumns[i]); + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + aTblColumns[i].name + ); + const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId + }); + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name) + ?.id; + await sMap.addToMappingTbl( + aTblColumns[i].id, + ncId, + aTblColumns[i].name, + ncTbl.id + ); + + syncLog(`NC API: dbTableColumn.create LOOKUP`); + } + } + } + + let level = 2; + let nestedCnt = 0; + while (nestedLookupTbl.length) { + // if nothing has changed from previous iteration, skip rest + if (nestedCnt === nestedLookupTbl.length) { + for (let i = 0; i < nestedLookupTbl.length; i++) { + const fTblField = + nestedLookupTbl[i].typeOptions.foreignTableRollupColumnId; + const name = aTbl_getColumnName(fTblField); + addToSkipColumnLog( + ncSchema.tablesById[nestedLookupTbl[i].srcTableId]?.title, + nestedLookupTbl[i].name, + nestedLookupTbl[i].type, + `foreign table field not found [${name.tn}/${name.cn}]` + ); + } + if (enableErrorLogs) + console.log( + `## Failed to configure ${nestedLookupTbl.length} lookups` + ); + break; + } + + // Nested lookup + nestedCnt = nestedLookupTbl.length; + for (let i = 0; i < nestedLookupTbl.length; i++) { + syncLog( + `Configuring Nested Lookup: Level-${level} [${i + 1}/${nestedCnt}]` + ); + + const srcTableId = nestedLookupTbl[0].srcTableId; + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + const ncRelationColumnId = sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.relationColumnId + ); + const ncLookupColumnId = sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId + ); + + if (ncLookupColumnId === undefined) { + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + nestedLookupTbl[0].name + ); + const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId + }); + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find( + x => x.title === nestedLookupTbl[0].name + )?.id; + await sMap.addToMappingTbl( + nestedLookupTbl[0].id, + ncId, + nestedLookupTbl[0].name, + ncTbl.id + ); + + // remove entry + nestedLookupTbl.splice(0, 1); + syncLog(`NC API: dbTableColumn.create LOOKUP`); + } + level++; + } + } + + function getRollupNcFunction(aTblFunction) { + const fn = aTblFunction.split('(')[0]; + const aTbl_ncRollUp = { + AND: '', + ARRAYCOMPACT: '', + ARRAYJOIN: '', + ARRAYUNIQUE: '', + AVERAGE: 'average', + CONCATENATE: '', + COUNT: 'count', + COUNTA: '', + COUNTALL: '', + MAX: 'max', + MIN: 'min', + OR: '', + SUM: 'sum', + XOR: '' + }; + return aTbl_ncRollUp[fn]; + } + + async function nocoCreateRollups(aTblSchema) { + // Rollups + for (let idx = 0; idx < aTblSchema.length; idx++) { + const aTblColumns = aTblSchema[idx].columns.filter( + x => x.type === 'rollup' + ); + + // parent table ID + // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + const srcTableId = sMap.getNcIdFromAtId(aTblSchema[idx].id); + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + if (aTblColumns.length) { + // rollup exist + for (let i = 0; i < aTblColumns.length; i++) { + syncLog( + `[${idx + 1}/${aTblSchema.length}] Configuring Rollup :: [${i + + 1}/${aTblColumns.length}] ${aTblSchema[idx].name}` + ); + + // fetch associated rollup function + // skip column creation if supported rollup function doesnot exist + const ncRollupFn = getRollupNcFunction( + aTblColumns[i].typeOptions.formulaTextParsed + ); + + if (ncRollupFn === '') { + addToSkipColumnLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + `rollup function ${aTblColumns[i].typeOptions.formulaTextParsed} not supported` + ); + continue; + } + + // something is not right, skip + if ( + aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + if (enableErrorLogs) + console.log(`## Invalid column IDs mapped; skip`); + + addToSkipColumnLog( + srcTableSchema.title, + aTblColumns[i].name, + aTblColumns[i].type, + 'invalid column ID in dependency list' + ); + continue; + } + + const ncRelationColumnId = sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.relationColumnId + ); + const ncRollupColumnId = sMap.getNcIdFromAtId( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + + if (ncRollupColumnId === undefined) { + aTblColumns[i]['srcTableId'] = srcTableId; + nestedRollupTbl.push(aTblColumns[i]); + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + aTblColumns[i].name + ); + const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + uidt: UITypes.Rollup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_rollup_column_id: ncRollupColumnId, + rollup_function: ncRollupFn + }); + updateNcTblSchema(ncTbl); + syncLog(`NC API: dbTableColumn.create ROLLUP`); + + const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name) + ?.id; + await sMap.addToMappingTbl( + aTblColumns[i].id, + ncId, + aTblColumns[i].name, + ncTbl.id + ); + } + } + } + syncLog(`Nested rollup: ${nestedRollupTbl.length}`); + } + + async function nocoLookupForRollups() { + const nestedCnt = nestedLookupTbl.length; + for (let i = 0; i < nestedLookupTbl.length; i++) { + syncLog(`Configuring Lookup over Rollup :: [${i + 1}/${nestedCnt}]`); + + const srcTableId = nestedLookupTbl[0].srcTableId; + const srcTableSchema = ncSchema.tablesById[srcTableId]; + + const ncRelationColumnId = sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.relationColumnId + ); + const ncLookupColumnId = sMap.getNcIdFromAtId( + nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId + ); + + if (ncLookupColumnId === undefined) { + continue; + } + + const ncName = nc_getSanitizedColumnName( + srcTableSchema, + nestedLookupTbl[0].name + ); + const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId + }); + updateNcTblSchema(ncTbl); + + const ncId = ncTbl.columns.find(x => x.title === nestedLookupTbl[0].name) + ?.id; + await sMap.addToMappingTbl( + nestedLookupTbl[0].id, + ncId, + nestedLookupTbl[0].name, + ncTbl.id + ); + + // remove entry + nestedLookupTbl.splice(0, 1); + syncLog(`NC API: dbTableColumn.create LOOKUP`); + } + } + + async function nocoSetPrimary(aTblSchema) { + for (let idx = 0; idx < aTblSchema.length; idx++) { + syncLog( + `[${idx + 1}/${aTblSchema.length}] Configuring Primary value : ${ + aTblSchema[idx].name + }` + ); + + const pColId = aTblSchema[idx].primaryColumnId; + const ncColId = sMap.getNcIdFromAtId(pColId); + + // skip primary column configuration if we field not migrated + syncLog(`NC API: dbTableColumn.primaryColumnSet`); + if (ncColId) { + await api.dbTableColumn.primaryColumnSet(ncColId); + + // update schema + const ncTblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); + await updateNcTblSchemaById(ncTblId); + } + } + } + + // retrieve nc-view column ID from corresponding nc-column ID + async function nc_getViewColumnId(viewId, viewType, ncColumnId) { + // retrieve view Info + let viewDetails = []; + + if (viewType === 'form') + viewDetails = (await api.dbView.formRead(viewId)).columns; + else if (viewType === 'gallery') + viewDetails = (await api.dbView.galleryRead(viewId)).columns; + else viewDetails = await api.dbView.gridColumnsList(viewId); + + const viewColumnId = viewDetails.find(x => x.fk_column_id === ncColumnId) + ?.id; + + return viewColumnId; + } + + // @ts-ignore + async function nc_hideColumn(tblName, viewName, hiddenColumns, viewType?) { + // retrieve table schema + const ncTbl = await nc_getTableSchema(tblName); + // retrieve view ID + const viewId = ncTbl.views.find(x => x.title === viewName).id; + + // retrieve view Info + let viewDetails = []; + + if (viewType === 'form') + viewDetails = (await api.dbView.formRead(viewId)).columns; + else if (viewType === 'gallery') + viewDetails = (await api.dbView.galleryRead(viewId)).columns; + else viewDetails = await api.dbView.gridColumnsList(viewId); + + for (let i = 0; i < hiddenColumns.length; i++) { + // retrieve column schema + const ncColumn = ncTbl.columns.find(x => x.title === hiddenColumns[i]); + // retrieve view column ID + const viewColumnId = viewDetails.find( + x => x.fk_column_id === ncColumn?.id + )?.id; + + // fix me + if (viewColumnId === undefined) { + if (enableErrorLogs) + console.log( + `## Column disable fail: ${tblName}, ${viewName}, ${hiddenColumns[i]}` + ); + continue; + } + + // hide + syncLog(`NC API: dbViewColumn.update ${viewId}, ${ncColumn.id}`); + // @ts-ignore + const retVal = await api.dbViewColumn.update(viewId, viewColumnId, { + show: false + }); + } + } + + ////////// Data processing + + async function nocoLinkProcessing(projName, table, record, _field) { + const rec = record.fields; + const refRowIdList: any = Object.values(rec); + const referenceColumnName = Object.keys(rec)[0]; + + if (refRowIdList.length) { + for (let i = 0; i < refRowIdList[0].length; i++) { + syncLog( + `NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}` + ); + + await api.dbTableRow.nestedAdd( + 'noco', + projName, + table.id, + `${record.id}`, + 'mm', // fix me + encodeURIComponent(referenceColumnName), + `${refRowIdList[0][i]}` + ); + } + } + } + + // fix me: + // instead of skipping data after retrieval, use select fields option in airtable API + async function nocoBaseDataProcessing(sDB, table, record) { + const recordHash = hash(record); + const rec = record.fields; + + // kludge - + // trim spaces on either side of column name + // leads to error in NocoDB + Object.keys(rec).forEach(key => { + const replacedKey = key.replace(/\?/g, 'QQ').trim(); + if (key !== replacedKey) { + rec[replacedKey] = rec[key]; + delete rec[key]; + } + }); + + // post-processing on the record + for (const [key, value] of Object.entries(rec as { [key: string]: any })) { + // retrieve datatype + const dt = table.columns.find(x => x.title === key)?.uidt; + + // if(dt === undefined) + // console.log('fix me') + + // https://www.npmjs.com/package/validator + // default value: digits_after_decimal: [2] + // if currency, set decimal place to 2 + // + if (dt === UITypes.Currency) rec[key] = (+value).toFixed(2); + + // we will pick up LTAR once all table data's are in place + if (dt === UITypes.LinkToAnotherRecord) { + aTblDataLinks.push(JSON.parse(JSON.stringify(rec))); + delete rec[key]; + } + + // these will be automatically populated depending on schema configuration + if (dt === UITypes.Lookup) delete rec[key]; + if (dt === UITypes.Rollup) delete rec[key]; + + if (dt === UITypes.Collaborator) { + // in case of multi-collaborator, this will be an array + if (Array.isArray(value)) { + let collaborators = ''; + for (let i = 0; i < value.length; i++) { + collaborators += `${value[i]?.name} <${value[i]?.email}>, `; + rec[key] = collaborators; + } + } else rec[key] = `${value?.name} <${value?.email}>`; + } + + if (dt === UITypes.Barcode) rec[key] = value.text; + if (dt === UITypes.Button) rec[key] = `${value?.label} <${value?.url}>`; + + if (dt === UITypes.DateTime) { + const atDateField = dayjs(value); + rec[key] = atDateField.utc().format('YYYY-MM-DD HH:mm'); + } + + if (dt === UITypes.Attachment) { + const tempArr = []; + for (const v of value) { + const binaryImage = await axios + .get(v.url, { + responseType: 'stream', + headers: { + 'Content-Type': v.type + } + }) + .then(response => { + return response.data; + }) + .catch(error => { + console.log(error); + return false; + }); + + const imageFile: any = new FormData(); + imageFile.append('files', binaryImage, { + filename: v.filename.includes('?') + ? v.filename.split('?')[0] + : v.filename + }); + + const rs = await axios + .post(sDB.baseURL + '/api/v1/db/storage/upload', imageFile, { + params: { + path: `noco/${sDB.projectName}/${table.title}/${key}` + }, + headers: { + 'Content-Type': `multipart/form-data; boundary=${imageFile._boundary}`, + 'xc-auth': sDB.authToken + } + }) + .then(response => { + return response.data; + }) + .catch(e => { + console.log(e); + }); + + tempArr.push(...rs); + } + rec[key] = JSON.stringify(tempArr); + } + } + + // insert airtable record ID explicitly into each records + rec['_aTbl_nc_rec_id'] = record.id; + rec['_aTbl_nc_rec_hash'] = recordHash; + + // console.log(rec) + + syncLog(`NC API: dbTableRow.bulkCreate ${table.title} [${rec}]`); + // console.log(JSON.stringify(rec, null, 2)) + + // bulk Insert + // @ts-ignore + const returnValue = await api.dbTableRow.bulkCreate( + 'nc', + sDB.projectName, + table.id, // encodeURIComponent(table.title), + [rec] + ); + } + + async function nocoReadData(sDB, table, callback) { + return new Promise((resolve, reject) => { + base(table.title) + .select({ + pageSize: 100 + // maxRecords: 1, + }) + .eachPage( + async function page(records, fetchNextPage) { + // console.log(JSON.stringify(records, null, 2)); + + // This function (`page`) will get called for each page of records. + await Promise.all( + records.map(record => callback(sDB, table, record)) + ); + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + function done(err) { + if (err) { + console.error(err); + reject(err); + } + resolve(null); + } + ); + }); + } + + async function nocoReadDataSelected(projName, table, callback, fields) { + return new Promise((resolve, reject) => { + base(table.title) + .select({ + pageSize: 100, + // maxRecords: 100, + fields: [fields] + }) + .eachPage( + async function page(records, fetchNextPage) { + // console.log(JSON.stringify(records, null, 2)); + + // This function (`page`) will get called for each page of records. + // records.forEach(record => callback(table, record)); + await Promise.all( + records.map(r => callback(projName, table, r, fields)) + ); + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + function done(err) { + if (err) { + console.error(err); + reject(err); + } + resolve(null); + } + ); + }); + } + + ////////// + + function nc_isLinkExists(atblFieldId) { + if ( + ncLinkMappingTable.find( + x => x.aTbl.typeOptions.symmetricColumnId === atblFieldId + ) + ) + return true; + return false; + } + + async function nocoCreateProject(projName) { + syncLog(`Create Project: ${projName}`); + + // create empty project (XC-DB) + ncCreatedProjectSchema = await api.project.create({ + title: projName + }); + } + + async function nocoGetProject(projId) { + syncLog(`Getting project meta: ${projId}`); + + // create empty project (XC-DB) + ncCreatedProjectSchema = await api.project.read(projId); + } + + async function nocoConfigureGalleryView(sDB, aTblSchema) { + if (!sDB.syncViews) return; + for (let idx = 0; idx < aTblSchema.length; idx++) { + const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + const galleryViews = aTblSchema[idx].views.filter( + x => x.type === 'gallery' + ); + + const configuredViews = + runTimeCounters.view.grid + + runTimeCounters.view.gallery + + runTimeCounters.view.form; + runTimeCounters.view.gallery += galleryViews.length; + + for (let i = 0; i < galleryViews.length; i++) { + syncLog( + `[${configuredViews + i + 1}/${ + runTimeCounters.view.total + }] Configuring view :: Gallery` + ); + syncLog(` Axios fetch view-data`); + + // create view + // @ts-ignore + const vData = await getViewData(galleryViews[i].id); + const viewName = aTblSchema[idx].views.find( + x => x.id === galleryViews[i].id + )?.name; + + syncLog(` Create NC View :: ${viewName}`); + // @ts-ignore + const g = await api.dbView.galleryCreate(tblId, { title: viewName }); + await updateNcTblSchemaById(tblId); + // syncLog(`[${idx+1}/${aTblSchema.length}][Gallery View][${i+1}/${galleryViews.length}] Create ${viewName}`) + + // await nc_configureFields(g.id, vData.columnOrder, aTblSchema[idx].name, viewName, 'gallery'); + } + } + } + + async function nocoConfigureFormView(sDB, aTblSchema) { + if (!sDB.syncViews) return; + for (let idx = 0; idx < aTblSchema.length; idx++) { + const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); + const formViews = aTblSchema[idx].views.filter(x => x.type === 'form'); + + const configuredViews = + runTimeCounters.view.grid + + runTimeCounters.view.gallery + + runTimeCounters.view.form; + runTimeCounters.view.form += formViews.length; + for (let i = 0; i < formViews.length; i++) { + syncLog( + `[${configuredViews + i + 1}/${ + runTimeCounters.view.total + }] Configuring view :: Form` + ); + syncLog(` Axios fetch view-data`); + + // create view + const vData = await getViewData(formViews[i].id); + const viewName = aTblSchema[idx].views.find( + x => x.id === formViews[i].id + )?.name; + + // everything is default + let refreshMode = 'NO_REFRESH'; + let msg = 'Thank you for submitting the form!'; + let desc = ''; + + // response will not include form object if everything is default + // + if (vData.metadata?.form) { + refreshMode = vData.metadata.form.refreshAfterSubmit; + msg = vData.metadata.form?.afterSubmitMessage + ? vData.metadata.form.afterSubmitMessage + : 'Thank you for submitting the form!'; + desc = vData.metadata.form.description; + } + + const formData = { + title: viewName, + heading: viewName, + subheading: desc, + success_msg: msg, + submit_another_form: refreshMode.includes('REFRESH_BUTTON') + ? true + : false, + show_blank_form: refreshMode.includes('AUTO_REFRESH') ? true : false + }; + + syncLog(` Create NC View :: ${viewName}`); + const f = await api.dbView.formCreate(tblId, formData); + syncLog( + `[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${ + formViews.length + }] Create ${viewName}` + ); + + await updateNcTblSchemaById(tblId); + + syncLog(` Configure show/hide columns`); + await nc_configureFields( + f.id, + vData.columnOrder, + aTblSchema[idx].name, + viewName, + 'form' + ); + } + } + } + + async function nocoConfigureGridView(sDB, aTblSchema) { + for (let idx = 0; idx < aTblSchema.length; idx++) { + const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); + const gridViews = aTblSchema[idx].views.filter(x => x.type === 'grid'); + + const configuredViews = + runTimeCounters.view.grid + + runTimeCounters.view.gallery + + runTimeCounters.view.form; + runTimeCounters.view.grid += gridViews.length; + + for (let i = 0; i < (sDB.syncViews ? gridViews.length : 1); i++) { + syncLog( + `[${configuredViews + i + 1}/${ + runTimeCounters.view.total + }] Configuring view :: Grid` + ); + syncLog(` Axios fetch view-data`); + // fetch viewData JSON + const vData = await getViewData(gridViews[i].id); + + // retrieve view name & associated NC-ID + const viewName = aTblSchema[idx].views.find( + x => x.id === gridViews[i].id + )?.name; + const viewList: any = await api.dbView.list(tblId); + const ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id; + + // create view (default already created) + if (i > 0) { + syncLog(` Create NC View :: ${viewName}`); + const viewCreated = await api.dbView.gridCreate(tblId, { + title: viewName + }); + await updateNcTblSchemaById(tblId); + await sMap.addToMappingTbl( + gridViews[i].id, + viewCreated.id, + viewName, + tblId + ); + // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Create ${viewName}`) + } + + // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Hide columns ${viewName}`) + syncLog(` Configure show/hide columns`); + await nc_configureFields( + ncViewId, + vData.columnOrder, + aTblSchema[idx].name, + viewName, + 'grid' + ); + + // configure filters + if (vData?.filters) { + // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure filters ${viewName}`) + syncLog(` Configure filter set`); + + // skip filters if nested + if (!vData.filters.filterSet.find(x => x?.type === 'nested')) { + await nc_configureFilters(ncViewId, vData.filters); + } + } + + // configure sort + if (vData?.lastSortsApplied?.sortSet.length) { + // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure sort ${viewName}`) + syncLog(` Configure sort set`); + await nc_configureSort(ncViewId, vData.lastSortsApplied); + } + } + } + } + + async function nocoAddUsers(aTblSchema) { + const userRoles = { + owner: 'owner', + create: 'creator', + edit: 'editor', + comment: 'commenter', + read: 'viewer', + none: 'viewer' + }; + const userList = aTblSchema.appBlanket.userInfoById; + const totalUsers = Object.keys(userList).length; + let cnt = 0; + + for (const [_key, value] of Object.entries( + userList as { [key: string]: any } + )) { + syncLog(`[${++cnt}/${totalUsers}] Configuring User :: ${value.email}`); + await api.auth.projectUserAdd(ncCreatedProjectSchema.id, { + email: value.email, + roles: userRoles[value.permissionLevel] + }); + } + } + + function updateNcTblSchema(tblSchema) { + const tblId = tblSchema.id; + + // replace entry from array if already exists + const idx = ncSchema.tables.findIndex(x => x.id === tblId); + if (idx !== -1) ncSchema.tables.splice(idx, 1); + ncSchema.tables.push(tblSchema); + + // overwrite object if it exists + ncSchema.tablesById[tblId] = tblSchema; + } + + async function updateNcTblSchemaById(tblId) { + const ncTbl = await api.dbTable.read(tblId); + updateNcTblSchema(ncTbl); + } + + // @ts-ignore + async function nocoReadNcSchema() { + const tableList = await api.dbTable.list(ncCreatedProjectSchema.id); + for (let tblCnt = 0; tblCnt < tableList.list.length; tblCnt++) { + const tblSchema = await api.dbTable.read(tableList.list[tblCnt].id); + updateNcTblSchema(tblSchema); + } + } + + /////////////////////// + + // statistics + // + const migrationStats = []; + + async function generateMigrationStats(aTblSchema) { + const migrationStatsObj = { + table_name: '', + aTbl: { + columns: 0, + links: 0, + lookup: 0, + rollup: 0 + }, + nc: { + columns: 0, + links: 0, + lookup: 0, + rollup: 0, + invalidColumn: 0 + } + }; + for (let idx = 0; idx < aTblSchema.length; idx++) { + migrationStatsObj.table_name = aTblSchema[idx].name; + + const aTblLinkColumns = aTblSchema[idx].columns.filter( + x => x.type === 'foreignKey' + ); + const aTblLookups = aTblSchema[idx].columns.filter( + x => x.type === 'lookup' + ); + const aTblRollups = aTblSchema[idx].columns.filter( + x => x.type === 'rollup' + ); + + let invalidColumnId = 0; + for (let i = 0; i < aTblLookups.length; i++) { + if ( + aTblLookups[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + invalidColumnId++; + } + } + for (let i = 0; i < aTblRollups.length; i++) { + if ( + aTblRollups[i]?.typeOptions?.dependencies?.invalidColumnIds?.length + ) { + invalidColumnId++; + } + } + + migrationStatsObj.aTbl.columns = aTblSchema[idx].columns.length; + migrationStatsObj.aTbl.links = aTblLinkColumns.length; + migrationStatsObj.aTbl.lookup = aTblLookups.length; + migrationStatsObj.aTbl.rollup = aTblRollups.length; + + const ncTbl = await nc_getTableSchema(aTblSchema[idx].name); + const linkColumn = ncTbl.columns.filter( + x => x.uidt === UITypes.LinkToAnotherRecord + ); + const lookup = ncTbl.columns.filter(x => x.uidt === UITypes.Lookup); + const rollup = ncTbl.columns.filter(x => x.uidt === UITypes.Rollup); + + // all links hardwired as m2m. m2m generates additional tables per link + // hence link/2 + migrationStatsObj.nc.columns = + ncTbl.columns.length - linkColumn.length / 2; + migrationStatsObj.nc.links = linkColumn.length / 2; + migrationStatsObj.nc.lookup = lookup.length; + migrationStatsObj.nc.rollup = rollup.length; + migrationStatsObj.nc.invalidColumn = invalidColumnId; + + const temp = JSON.parse(JSON.stringify(migrationStatsObj)); + migrationStats.push(temp); + } + + const columnSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.columns; + }, 0); + const linkSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.links; + }, 0); + const lookupSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.lookup; + }, 0); + const rollupSum = migrationStats.reduce((accumulator, object) => { + return accumulator + object.nc.rollup; + }, 0); + + syncLog(`Quick Stats:`); + syncLog(` Total Tables: ${aTblSchema.length}`); + syncLog(` Total Columns: ${columnSum}`); + syncLog(` Links: ${linkSum}`); + syncLog(` Lookup: ${lookupSum}`); + syncLog(` Rollup: ${rollupSum}`); + syncLog(` Total Filters: ${runTimeCounters.filter}`); + syncLog(` Total Sort: ${runTimeCounters.sort}`); + syncLog(` Total Views: ${runTimeCounters.view.total}`); + syncLog(` Grid: ${runTimeCounters.view.grid}`); + syncLog(` Gallery: ${runTimeCounters.view.gallery}`); + syncLog(` Form: ${runTimeCounters.view.form}`); + + const duration = Date.now() - start; + syncLog(`Migration time: ${duration}`); + syncLog(`Axios fetch count: ${runTimeCounters.fetchAt.count}`); + syncLog(`Axios fetch time: ${runTimeCounters.fetchAt.time}`); + } + + ////////////////////////////// + // filters + + const filterMap = { + '=': 'eq', + '!=': 'neq', + '<': 'lt', + '<=': 'lte', + '>': 'gt', + '>=': 'gte', + isEmpty: 'empty', + isNotEmpty: 'notempty', + contains: 'like', + doesNotContain: 'nlike', + isAnyOf: 'eq', + isNoneOf: 'neq' + }; + + async function nc_configureFilters(viewId, f) { + for (let i = 0; i < f.filterSet.length; i++) { + const filter = f.filterSet[i]; + const colSchema = await nc_getColumnSchema(filter.columnId); + + // column not available; + // one of not migrated column; + if (!colSchema) { + addUserInfo( + `Filter configuration partial: ${sMap.getNcNameFromAtId(viewId)}` + ); + continue; + } + + const columnId = colSchema.id; + const datatype = colSchema.uidt; + const ncFilters = []; + + // console.log(filter) + if (datatype === UITypes.Date) { + // skip filters over data datatype + addUserInfo( + `Filter configuration partial: ${sMap.getNcNameFromAtId(viewId)}` + ); + continue; + } + + // single-select & multi-select + else if ( + datatype === UITypes.SingleSelect || + datatype === UITypes.MultiSelect + ) { + // if array, break it down to multiple filters + if (Array.isArray(filter.value)) { + for (let i = 0; i < filter.value.length; i++) { + const fx = { + fk_column_id: columnId, + logical_op: f.conjunction, + comparison_op: filterMap[filter.operator], + value: sMap.getNcNameFromAtId(filter.value[i]) + }; + ncFilters.push(fx); + } + } + // not array - add as is + else if (filter.value) { + const fx = { + fk_column_id: columnId, + logical_op: f.conjunction, + comparison_op: filterMap[filter.operator], + value: sMap.getNcNameFromAtId(filter.value) + }; + ncFilters.push(fx); + } + } + + // other data types (number/ text/ long text/ ..) + else if (filter.value) { + const fx = { + fk_column_id: columnId, + logical_op: f.conjunction, + comparison_op: filterMap[filter.operator], + value: filter.value + }; + ncFilters.push(fx); + } + + // insert filters + for (let i = 0; i < ncFilters.length; i++) { + await api.dbTableFilter.create(viewId, { + ...ncFilters[i] + }); + runTimeCounters.filter++; + } + } + } + + async function nc_configureSort(viewId, s) { + for (let i = 0; i < s.sortSet.length; i++) { + const columnId = (await nc_getColumnSchema(s.sortSet[i].columnId))?.id; + + if (columnId) + await api.dbTableSort.create(viewId, { + fk_column_id: columnId, + direction: s.sortSet[i].ascending ? 'asc' : 'dsc' + }); + runTimeCounters.sort++; + } + } + + async function nc_configureFields(_viewId, c, tblName, viewName, viewType?) { + // force hide PK column + const hiddenColumns = ['_aTbl_nc_rec_id', '_aTbl_nc_rec_hash']; + + // // extract other columns hidden in this view + // const hiddenColumnID = c.filter(x => x.visibility === false); + // for (let j = 0; j < hiddenColumnID.length; j++) { + // hiddenColumns.push(aTbl_getColumnName(hiddenColumnID[j].columnId).cn); + // } + + // await nc_hideColumn(tblName, viewName, hiddenColumns, viewType); + + // column order corrections + // retrieve table schema + const ncTbl = await nc_getTableSchema(tblName); + // retrieve view ID + const viewId = ncTbl.views.find(x => x.title === viewName).id; + + // nc-specific columns; default hide. + for (let j = 0; j < hiddenColumns.length; j++) { + const ncColumnId = ncTbl.columns.find(x => x.title === hiddenColumns[j]) + .id; + const ncViewColumnId = await nc_getViewColumnId( + viewId, + viewType, + ncColumnId + ); + if (ncViewColumnId === undefined) continue; + + // first two positions held by record id & record hash + await api.dbViewColumn.update(viewId, ncViewColumnId, { + show: false, + order: j + 1 + c.length + }); + } + + // rest of the columns from airtable- retain order & visibility property + for (let j = 0; j < c.length; j++) { + const ncColumnId = sMap.getNcIdFromAtId(c[j].columnId); + const ncViewColumnId = await nc_getViewColumnId( + viewId, + viewType, + ncColumnId + ); + if (ncViewColumnId === undefined) continue; + + // first two positions held by record id & record hash + await api.dbViewColumn.update(viewId, ncViewColumnId, { + show: c[j].visibility, + order: j + 1 + }); + } + } + + /////////////////////////////////////////////////////////////////////////////// + const userInfo = []; + + function addUserInfo(log) { + userInfo.push(log); + } + + try { + syncLog = progress; + progress('SDK initialized'); + api = new Api({ + baseURL: syncDB.baseURL, + headers: { + 'xc-auth': syncDB.authToken + } + }); + + progress('Project initialization started'); + // delete project if already exists + if (debugMode) await init(syncDB); + + progress('Project initialized'); + + progress('Project schema extraction started'); + // read schema file + const schema = await getAtableSchema(syncDB); + const aTblSchema = schema.tableSchemas; + progress('Project schema extraction completed'); + + if (!syncDB.projectId) { + if (!syncDB.projectName) + throw new Error('Project name or id not provided'); + // create empty project + await nocoCreateProject(syncDB.projectName); + progress('Project created'); + } else { + await nocoGetProject(syncDB.projectId); + syncDB.projectName = ncCreatedProjectSchema?.title; + progress('Getting existing project meta'); + } + + progress('Table creation started'); + // prepare table schema (base) + await nocoCreateBaseSchema(aTblSchema); + progress('Table creation completed'); + + progress('Migrating LTAR columns'); + // add LTAR + await nocoCreateLinkToAnotherRecord(aTblSchema); + progress('Migrating LTAR columns completed'); + + progress('Migrating Lookup columns'); + // add look-ups + await nocoCreateLookups(aTblSchema); + progress('Migrating Lookup columns completed'); + + progress('Migrating Rollup columns'); + // add roll-ups + await nocoCreateRollups(aTblSchema); + progress('Migrating Rollup columns completed'); + + progress('Migrating Lookup form Rollup columns'); + // lookups for rollups + await nocoLookupForRollups(); + progress('Migrating Lookup form Rollup columns completed'); + + progress('Configuring primary value column'); + // configure primary values + await nocoSetPrimary(aTblSchema); + progress('Configuring primary value column completed'); + + progress('Adding users'); + // add users + await nocoAddUsers(schema); + progress('Adding users completed'); + + // hide-fields + // await nocoReconfigureFields(aTblSchema); + + progress('Syncing views'); + // configure views + await nocoConfigureGridView(syncDB, aTblSchema); + await nocoConfigureFormView(syncDB, aTblSchema); + await nocoConfigureGalleryView(syncDB, aTblSchema); + progress('Syncing views completed'); + + if (process_aTblData) { + try { + // await nc_DumpTableSchema(); + const ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); + for (let i = 0; i < ncTblList.list.length; i++) { + const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + progress(`Reading data from ${ncTbl.title}`); + let c = 0; + await nocoReadData(syncDB, ncTbl, async (sDB, table, record) => { + progress( + `Processing records from ${ncTbl.title} : ${c} - ${(c += 25)}` + ); + await nocoBaseDataProcessing(sDB, table, record); + }); + progress(`Data inserted from ${ncTbl.title}`); + } + + // Configure link @ Data row's + for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { + const x = ncLinkMappingTable[idx]; + const ncTbl = await nc_getTableSchema( + aTbl_getTableName(x.aTbl.tblId).tn + ); + progress(`Linking data to ${ncTbl.title}`); + let c = 0; + await nocoReadDataSelected( + syncDB.projectName, + ncTbl, + async (projName, table, record, _field) => { + progress( + `Mapping LTAR records from ${ncTbl.title} : ${c} - ${(c += 25)}` + ); + await nocoLinkProcessing(projName, table, record, _field); + }, + x.aTbl.name + ); + progress(`Linked data to ${ncTbl.title}`); + } + } catch (error) { + progress(`There was an error while migrating data! Please make sure your API key (${syncDB.apiKey}) is correct.`); + progress(`Error: ${error}`); + } + } + if (generate_migrationStats) { + await generateMigrationStats(aTblSchema); + } + } catch (e) { + if (e.response?.data?.msg) { + throw new Error(e.response.data.msg); + } + throw e; + } +}; + +export interface AirtableSyncConfig { + id: string; + baseURL: string; + authToken: string; + projectName?: string; + projectId?: string; + apiKey: string; + shareId: string; + syncViews: boolean; +} diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/syncMap.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/syncMap.ts new file mode 100644 index 0000000000..20c3375a94 --- /dev/null +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/syncMap.ts @@ -0,0 +1,31 @@ +export const mapTbl = {}; + +// static mapping records between aTblId && ncId +export const addToMappingTbl = function addToMappingTbl( + aTblId, + ncId, + ncName, + parent? +) { + mapTbl[aTblId] = { + ncId: ncId, + ncParent: parent, + // name added to assist in quick debug + ncName: ncName + }; +}; + +// get NcID from airtable ID +export const getNcIdFromAtId = function getNcIdFromAtId(aId) { + return mapTbl[aId]?.ncId; +}; + +// get nc Parent from airtable ID +export const getNcParentFromAtId = function getNcParentFromAtId(aId) { + return mapTbl[aId]?.ncParent; +}; + +// get nc-title from airtable ID +export const getNcNameFromAtId = function getNcNameFromAtId(aId) { + return mapTbl[aId]?.ncName; +}; diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/importApis.ts b/packages/nocodb/src/lib/noco/meta/api/sync/importApis.ts new file mode 100644 index 0000000000..5855f48e8e --- /dev/null +++ b/packages/nocodb/src/lib/noco/meta/api/sync/importApis.ts @@ -0,0 +1,79 @@ +import { Request, Router } from 'express'; +// import { Queue } from 'bullmq'; +// import axios from 'axios'; +import catchError from '../../helpers/catchError'; +import { Socket } from 'socket.io'; +import NocoJobs from '../../../../noco-jobs/NocoJobs'; +import job, { AirtableSyncConfig } from './helpers/job'; +import SyncSource from '../../../../noco-models/SyncSource'; +import Noco from '../../../Noco'; +import * as jwt from 'jsonwebtoken'; +const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB'; + +enum SyncStatus { + PROGRESS = 'PROGRESS', + COMPLETED = 'COMPLETED', + FAILED = 'FAILED' +} + +export default (router: Router, clients: { [id: string]: Socket }) => { + NocoJobs.jobsMgr.addJobWorker(AIRTABLE_IMPORT_JOB, job); + NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => { + clients?.[payload?.id]?.emit('progress', { + msg: progress, + status: SyncStatus.PROGRESS + }); + }); + NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, payload => { + clients?.[payload?.id]?.emit('progress', { + msg: 'completed', + status: SyncStatus.COMPLETED + }); + }); + NocoJobs.jobsMgr.addFailureCbk(AIRTABLE_IMPORT_JOB, (payload, error: any) => { + clients?.[payload?.id]?.emit('progress', { + msg: error?.message || 'Failed due to some internal error', + status: SyncStatus.FAILED + }); + }); + + router.post( + '/api/v1/db/meta/import/airtable', + catchError((req, res) => { + NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { + id: req.query.id, + ...req.body + }); + res.json({}); + }) + ); + router.post( + '/api/v1/db/meta/syncs/:syncId/trigger', + catchError(async (req: Request, res) => { + const syncSource = await SyncSource.get(req.params.syncId); + + const user = await syncSource.getUser(); + const token = jwt.sign( + { + email: user.email, + firstname: user.firstname, + lastname: user.lastname, + id: user.id, + roles: user.roles + }, + + Noco.getConfig().auth.jwt.secret, + Noco.getConfig().auth.jwt.options + ); + + NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { + id: req.query.id, + ...(syncSource?.details || {}), + projectId: syncSource.project_id, + authToken: token, + baseURL: (req as any).ncSiteUrl + }); + res.json({}); + }) + ); +}; diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/syncSourceApis.ts b/packages/nocodb/src/lib/noco/meta/api/sync/syncSourceApis.ts new file mode 100644 index 0000000000..ce2d00b1af --- /dev/null +++ b/packages/nocodb/src/lib/noco/meta/api/sync/syncSourceApis.ts @@ -0,0 +1,53 @@ +import { Request, Response, Router } from 'express'; + +import { Tele } from 'nc-help'; +import SyncSource from '../../../../noco-models/SyncSource'; +import { PagedResponseImpl } from '../../helpers/PagedResponse'; +import ncMetaAclMw from '../../helpers/ncMetaAclMw'; + +export async function syncSourceList(req: Request, res: Response) { + // todo: pagination + res.json(new PagedResponseImpl(await SyncSource.list(req.params.projectId))); +} + +export async function syncCreate(req: Request, res: Response) { + Tele.emit('evt', { evt_type: 'webhooks:created' }); + const sync = await SyncSource.insert({ + ...req.body, + fk_user_id: (req as any).user.id, + project_id: req.params.projectId + }); + res.json(sync); +} + +export async function syncDelete(req: Request, res: Response) { + Tele.emit('evt', { evt_type: 'webhooks:deleted' }); + res.json(await SyncSource.delete(req.params.syncId)); +} + +export async function syncUpdate(req: Request, res: Response) { + Tele.emit('evt', { evt_type: 'webhooks:updated' }); + + res.json(await SyncSource.update(req.params.syncId, req.body)); +} + +const router = Router({ mergeParams: true }); + +router.get( + '/api/v1/db/meta/projects/:projectId/syncs', + ncMetaAclMw(syncSourceList, 'syncSourceList') +); +router.post( + '/api/v1/db/meta/projects/:projectId/syncs', + ncMetaAclMw(syncCreate, 'syncSourceCreate') +); +router.delete( + '/api/v1/db/meta/syncs/:syncId', + ncMetaAclMw(syncDelete, 'syncSourceDelete') +); +router.patch( + '/api/v1/db/meta/syncs/:syncId', + ncMetaAclMw(syncUpdate, 'syncSourceUpdate') +); + +export default router; diff --git a/packages/nocodb/src/lib/noco/meta/helpers/extractProps.ts b/packages/nocodb/src/lib/noco/meta/helpers/extractProps.ts index 266fbbe84a..ba4aef7bb2 100644 --- a/packages/nocodb/src/lib/noco/meta/helpers/extractProps.ts +++ b/packages/nocodb/src/lib/noco/meta/helpers/extractProps.ts @@ -1,4 +1,5 @@ export default function extractProps(body: T, props: string[]): Partial { + // todo: throw error if no props found return props.reduce((o, key) => { if (key in body) o[key] = body[key]; return o; diff --git a/packages/nocodb/src/lib/noco/migrationsv2/nc_013_sync_source.ts b/packages/nocodb/src/lib/noco/migrationsv2/nc_013_sync_source.ts new file mode 100644 index 0000000000..638eab0eec --- /dev/null +++ b/packages/nocodb/src/lib/noco/migrationsv2/nc_013_sync_source.ts @@ -0,0 +1,72 @@ +import Knex from 'knex'; +import { MetaTable } from '../../utils/globals'; + +const up = async (knex: Knex) => { + await knex.schema.createTable(MetaTable.SYNC_SOURCE, table => { + table + .string('id', 20) + .primary() + .notNullable(); + + table.string('title'); + table.string('type'); + table.text('details'); + table.boolean('deleted'); + table.boolean('enabled').defaultTo(true); + table.float('order'); + + table.string('project_id', 128); + table.foreign('project_id').references(`${MetaTable.PROJECT}.id`); + table.string('fk_user_id', 128); + table.foreign('fk_user_id').references(`${MetaTable.USERS}.id`); + + table.timestamps(true, true); + }); + + await knex.schema.createTable(MetaTable.SYNC_LOGS, table => { + table + .string('id', 20) + .primary() + .notNullable(); + + table.string('project_id', 128); + table.string('fk_sync_source_id', 20); + // table + // .foreign('fk_sync_source_id') + // .references(`${MetaTable.SYNC_SOURCE}.id`); + + table.integer('time_taken'); + table.string('status'); + table.text('status_details'); + + table.timestamps(true, true); + }); +}; + +const down = async knex => { + await knex.schema.dropTable(MetaTable.SYNC_SOURCE); +}; + +export { up, down }; + +/** + * @copyright Copyright (c) 2022, Xgene Cloud Ltd + * + * @author Wing-Kam Wong + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ diff --git a/packages/nocodb/src/lib/utils/globals.ts b/packages/nocodb/src/lib/utils/globals.ts index eada1fb767..e34c0ab296 100644 --- a/packages/nocodb/src/lib/utils/globals.ts +++ b/packages/nocodb/src/lib/utils/globals.ts @@ -34,7 +34,9 @@ export enum MetaTable { PLUGIN = 'nc_plugins_v2', PROJECT_USERS = 'nc_project_users_v2', MODEL_ROLE_VISIBILITY = 'nc_disabled_models_for_role_v2', - API_TOKENS = 'nc_api_tokens' + API_TOKENS = 'nc_api_tokens', + SYNC_SOURCE = 'nc_sync_source_v2', + SYNC_LOGS = 'nc_sync_logs_v2' } export enum CacheScope {