feat: multiple base handle import & sync

Signed-off-by: mertmit <mertmit99@gmail.com>

packages/nc-gui/components/dashboard/TreeView.vue

@@ -145,7 +145,7 @@ const addTableTab = (table: TableType) => {
   addTab({ title: table.title, id: table.id, type: table.type as TabType })
 }
 
-function openRenameTableDialog(table: TableType, baseId: string, rightClick = false) {
+function openRenameTableDialog(table: TableType, baseId?: string, rightClick = false) {
   $e(rightClick ? 'c:table:rename:navdraw:right-click' : 'c:table:rename:navdraw:options')
 
   const isOpen = ref(true)
@@ -153,7 +153,7 @@ function openRenameTableDialog(table: TableType, baseId: string, rightClick = fa
   const { close } = useDialog(resolveComponent('DlgTableRename'), {
     'modelValue': isOpen,
     'tableMeta': table,
-    'baseId': baseId,
+    'baseId': baseId || bases.value[0].id,
     'onUpdate:modelValue': closeDialog,
   })
 
@@ -164,7 +164,7 @@ function openRenameTableDialog(table: TableType, baseId: string, rightClick = fa
   }
 }
 
-function openQuickImportDialog(type: string) {
+function openQuickImportDialog(type: string, baseId?: string) {
   $e(`a:actions:import-${type}`)
 
   const isOpen = ref(true)
@@ -172,6 +172,7 @@ function openQuickImportDialog(type: string) {
   const { close } = useDialog(resolveComponent('DlgQuickImport'), {
     'modelValue': isOpen,
     'importType': type,
+    'baseId': baseId || bases.value[0].id,
     'onUpdate:modelValue': closeDialog,
   })
 
@@ -182,13 +183,14 @@ function openQuickImportDialog(type: string) {
   }
 }
 
-function openAirtableImportDialog() {
+function openAirtableImportDialog(baseId?: string) {
   $e('a:actions:import-airtable')
 
   const isOpen = ref(true)
 
   const { close } = useDialog(resolveComponent('DlgAirtableImport'), {
     'modelValue': isOpen,
+    'baseId': baseId || bases.value[0].id,
     'onUpdate:modelValue': closeDialog,
   })
 
@@ -200,15 +202,14 @@ function openAirtableImportDialog() {
 }
 
 function openTableCreateDialog(baseId?: string) {
-  if (!baseId) return
   $e('c:table:create:navdraw')
 
   const isOpen = ref(true)
 
   const { close } = useDialog(resolveComponent('DlgTableCreate'), {
     'modelValue': isOpen,
+    'baseId': baseId || bases.value[0].id,
     'onUpdate:modelValue': closeDialog,
-    'baseId': baseId,
   })
 
   function closeDialog() {
@@ -301,7 +302,7 @@ useEventListener(document, 'keydown', async (e: KeyboardEvent) => {
                     <a-menu-item
                       v-if="isUIAllowed('airtableImport')"
                       key="quick-import-airtable"
-                      @click="openAirtableImportDialog"
+                      @click="openAirtableImportDialog(bases[0].id)"
                     >
                       <div class="color-transition nc-project-menu-item group">
                         <MdiTableLarge class="group-hover:text-accent" />
@@ -309,14 +310,22 @@ useEventListener(document, 'keydown', async (e: KeyboardEvent) => {
                       </div>
                     </a-menu-item>
 
-                    <a-menu-item v-if="isUIAllowed('csvImport')" key="quick-import-csv" @click="openQuickImportDialog('csv')">
+                    <a-menu-item
+                      v-if="isUIAllowed('csvImport')"
+                      key="quick-import-csv"
+                      @click="openQuickImportDialog('csv', bases[0].id)"
+                    >
                       <div class="color-transition nc-project-menu-item group">
                         <MdiFileDocumentOutline class="group-hover:text-accent" />
                         CSV file
                       </div>
                     </a-menu-item>
 
-                    <a-menu-item v-if="isUIAllowed('jsonImport')" key="quick-import-json" @click="openQuickImportDialog('json')">
+                    <a-menu-item
+                      v-if="isUIAllowed('jsonImport')"
+                      key="quick-import-json"
+                      @click="openQuickImportDialog('json', bases[0].id)"
+                    >
                       <div class="color-transition nc-project-menu-item group">
                         <MdiCodeJson class="group-hover:text-accent" />
                         JSON file
@@ -326,7 +335,7 @@ useEventListener(document, 'keydown', async (e: KeyboardEvent) => {
                     <a-menu-item
                       v-if="isUIAllowed('excelImport')"
                       key="quick-import-excel"
-                      @click="openQuickImportDialog('excel')"
+                      @click="openQuickImportDialog('excel', bases[0].id)"
                     >
                       <div class="color-transition nc-project-menu-item group">
                         <MdiFileExcel class="group-hover:text-accent" />
@@ -460,7 +469,7 @@ useEventListener(document, 'keydown', async (e: KeyboardEvent) => {
                               <a-menu-item
                                 v-if="isUIAllowed('airtableImport')"
                                 key="quick-import-airtable"
-                                @click="openAirtableImportDialog"
+                                @click="openAirtableImportDialog(base.id)"
                               >
                                 <div class="color-transition nc-project-menu-item group">
                                   <MdiTableLarge class="group-hover:text-accent" />
@@ -471,7 +480,7 @@ useEventListener(document, 'keydown', async (e: KeyboardEvent) => {
                               <a-menu-item
                                 v-if="isUIAllowed('csvImport')"
                                 key="quick-import-csv"
-                                @click="openQuickImportDialog('csv')"
+                                @click="openQuickImportDialog('csv', base.id)"
                               >
                                 <div class="color-transition nc-project-menu-item group">
                                   <MdiFileDocumentOutline class="group-hover:text-accent" />
@@ -482,7 +491,7 @@ useEventListener(document, 'keydown', async (e: KeyboardEvent) => {
                               <a-menu-item
                                 v-if="isUIAllowed('jsonImport')"
                                 key="quick-import-json"
-                                @click="openQuickImportDialog('json')"
+                                @click="openQuickImportDialog('json', base.id)"
                               >
                                 <div class="color-transition nc-project-menu-item group">
                                   <MdiCodeJson class="group-hover:text-accent" />
@@ -493,7 +502,7 @@ useEventListener(document, 'keydown', async (e: KeyboardEvent) => {
                               <a-menu-item
                                 v-if="isUIAllowed('excelImport')"
                                 key="quick-import-excel"
-                                @click="openQuickImportDialog('excel')"
+                                @click="openQuickImportDialog('excel', base.id)"
                               >
                                 <div class="color-transition nc-project-menu-item group">
                                   <MdiFileExcel class="group-hover:text-accent" />

packages/nc-gui/components/dlg/AirtableImport.vue

@@ -18,8 +18,9 @@ import {
   watch,
 } from '#imports'
 
-const { modelValue } = defineProps<{
+const { modelValue, baseId } = defineProps<{
   modelValue: boolean
+  baseId: string
 }>()
 
 const emit = defineEmits(['update:modelValue'])
@@ -100,7 +101,7 @@ async function createOrUpdate() {
         body: payload,
       })
     } else {
-      syncSource.value = await $fetch(`/api/v1/db/meta/projects/${project.value.id}/syncs`, {
+      syncSource.value = await $fetch(`/api/v1/db/meta/projects/${project.value.id}/syncs/${baseId}`, {
         baseURL,
         method: 'POST',
         headers: { 'xc-auth': $state.token.value as string },
@@ -113,7 +114,7 @@ async function createOrUpdate() {
 }
 
 async function loadSyncSrc() {
-  const data: any = await $fetch(`/api/v1/db/meta/projects/${project.value.id}/syncs`, {
+  const data: any = await $fetch(`/api/v1/db/meta/projects/${project.value.id}/syncs/${baseId}`, {
     baseURL,
     method: 'GET',
     headers: { 'xc-auth': $state.token.value as string },

packages/nc-gui/components/dlg/QuickImport.vue

@@ -27,6 +27,7 @@ import type { importFileList, streamImportFileList } from '~/lib'
 interface Props {
   modelValue: boolean
   importType: 'csv' | 'json' | 'excel'
+  baseId: string
   importDataOnly?: boolean
 }
 
@@ -364,6 +365,7 @@ const beforeUpload = (file: UploadFile) => {
             :import-data-only="importDataOnly"
             :quick-import-type="importType"
             :max-rows-to-parse="importState.parserConfig.maxRowsToParse"
+            :base-id="baseId"
             class="nc-quick-import-template-editor"
             @import="handleImport"
           />

packages/nc-gui/components/template/Editor.vue

@@ -30,7 +30,8 @@ import {
 } from '#imports'
 import { TabType } from '~/lib'
 
-const { quickImportType, projectTemplate, importData, importColumns, importDataOnly, maxRowsToParse } = defineProps<Props>()
+const { quickImportType, projectTemplate, importData, importColumns, importDataOnly, maxRowsToParse, baseId } =
+  defineProps<Props>()
 
 const emit = defineEmits(['import'])
 
@@ -45,6 +46,7 @@ interface Props {
   importColumns: any[]
   importDataOnly: boolean
   maxRowsToParse: number
+  baseId: string
 }
 
 interface Option {
@@ -397,7 +399,7 @@ async function importTemplate() {
     try {
       isImporting.value = true
 
-      const tableName = meta.value?.title
+      const tableId = meta.value?.id
       const projectName = project.value.title!
 
       await Promise.all(
@@ -436,8 +438,8 @@ async function importTemplate() {
                   return res
                 }, {}),
               )
-              await $api.dbTableRow.bulkCreate('noco', projectName, tableName!, batchData)
+              await $api.dbTableRow.bulkCreate('noco', projectName, tableId!, batchData)
-              updateImportTips(projectName, tableName!, progress, total)
+              updateImportTips(projectName, tableId!, progress, total)
               progress += batchData.length
             }
           })(key),
@@ -497,8 +499,7 @@ async function importTemplate() {
             }
           }
         }
-
+        const tableMeta = await $api.base.tableCreate(project?.value?.id as string, baseId as string, {
-        const tableMeta = await $api.dbTable.create(project?.value?.id as string, {
           table_name: table.table_name,
           // leave title empty to get a generated one based on table_name
           title: '',
@@ -534,7 +535,7 @@ async function importTemplate() {
                 for (let i = 0; i < data.length; i += offset) {
                   updateImportTips(projectName, tableMeta.title, progress, total)
                   const batchData = remapColNames(data.slice(i, i + offset), tableMeta.columns)
-                  await $api.dbTableRow.bulkCreate('noco', projectName, tableMeta.title, batchData)
+                  await $api.dbTableRow.bulkCreate('noco', projectName, tableMeta.id, batchData)
                   progress += batchData.length
                 }
                 updateImportTips(projectName, tableMeta.title, total, total)

packages/nocodb-sdk/src/lib/Api.ts

@@ -1988,6 +1988,32 @@ export class Api<
         ...params,
       }),
 
+    /**
+     * No description
+     *
+     * @tags Base
+     * @name TableList
+     * @request GET:/api/v1/db/meta/projects/{projectId}/{baseId}/tables
+     * @response `200` `TableListType`
+     */
+    tableList: (
+      projectId: string,
+      baseId: string,
+      query?: {
+        page?: number;
+        pageSize?: number;
+        sort?: string;
+        includeM2M?: boolean;
+      },
+      params: RequestParams = {}
+    ) =>
+      this.request<TableListType, any>({
+        path: `/api/v1/db/meta/projects/${projectId}/${baseId}/tables`,
+        method: 'GET',
+        query: query,
+        ...params,
+      }),
+
     /**
      * No description
      *

packages/nocodb/src/lib/meta/api/sync/helpers/job.ts

@@ -214,7 +214,7 @@ export default async (
   }
 
   function getRootDbType() {
-    return ncCreatedProjectSchema?.bases[0]?.type;
+    return ncCreatedProjectSchema?.bases.find((el) => el.id === syncDB.baseId)?.type;
   }
 
   // base mapping table
@@ -312,7 +312,7 @@ export default async (
   // @ts-ignore
   async function nc_DumpTableSchema() {
     console.log('[');
-    const ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id);
+    const ncTblList = await api.base.tableList(ncCreatedProjectSchema.id, syncDB.baseId);
     for (let i = 0; i < ncTblList.list.length; i++) {
       const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
       console.log(JSON.stringify(ncTbl, null, 2));
@@ -611,11 +611,12 @@ export default async (
     for (let idx = 0; idx < tables.length; idx++) {
       logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`);
 
-      logDetailed(`NC API: dbTable.create ${tables[idx].title}`);
+      logDetailed(`NC API: base.tableCreate ${tables[idx].title}`);
 
       let _perfStart = recordPerfStart();
-      const table: any = await api.dbTable.create(
+      const table: any = await api.base.tableCreate(
         ncCreatedProjectSchema.id,
+        syncDB.baseId,
         tables[idx]
       );
       recordPerfStats(_perfStart, 'dbTable.create');
@@ -2171,6 +2172,7 @@ export default async (
     } else {
       await nocoGetProject(syncDB.projectId);
       syncDB.projectName = ncCreatedProjectSchema?.title;
+      syncDB.baseId = syncDB.baseId || ncCreatedProjectSchema.bases[0].id;
       logDetailed('Getting existing project meta');
     }
 
@@ -2228,8 +2230,8 @@ export default async (
       try {
         // await nc_DumpTableSchema();
         const _perfStart = recordPerfStart();
-        const ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id);
+        const ncTblList = await api.base.tableList(ncCreatedProjectSchema.id, syncDB.baseId);
-        recordPerfStats(_perfStart, 'dbTable.list');
+        recordPerfStats(_perfStart, 'base.tableList');
 
         logBasic('Reading Records...');
 
@@ -2385,6 +2387,7 @@ export interface AirtableSyncConfig {
   authToken: string;
   projectName?: string;
   projectId?: string;
+  baseId?: string;
   apiKey: string;
   shareId: string;
   options: {

packages/nocodb/src/lib/meta/api/sync/importApis.ts

@@ -111,6 +111,7 @@ export default (
         id: req.params.syncId,
         ...(syncSource?.details || {}),
         projectId: syncSource.project_id,
+        baseId: syncSource.base_id,
         authToken: token,
         baseURL,
       });

packages/nocodb/src/lib/meta/api/sync/syncSourceApis.ts

@@ -4,17 +4,21 @@ import SyncSource from '../../../models/SyncSource';
 import { Tele } from 'nc-help';
 import { PagedResponseImpl } from '../../helpers/PagedResponse';
 import ncMetaAclMw from '../../helpers/ncMetaAclMw';
+import Project from '../../../models/Project';
 
 export async function syncSourceList(req: Request, res: Response) {
   // todo: pagination
-  res.json(new PagedResponseImpl(await SyncSource.list(req.params.projectId)));
+  res.json(new PagedResponseImpl(await SyncSource.list(req.params.projectId, req.params.baseId)));
 }
 
 export async function syncCreate(req: Request, res: Response) {
   Tele.emit('evt', { evt_type: 'webhooks:created' });
+  const project = await Project.getWithInfo(req.params.projectId);
+  
   const sync = await SyncSource.insert({
     ...req.body,
     fk_user_id: (req as any).user.id,
+    base_id: req.params.baseId ? req.params.baseId : project.bases[0].id,
     project_id: req.params.projectId,
   });
   res.json(sync);
@@ -41,6 +45,14 @@ router.post(
   '/api/v1/db/meta/projects/:projectId/syncs',
   ncMetaAclMw(syncCreate, 'syncSourceCreate')
 );
+router.get(
+  '/api/v1/db/meta/projects/:projectId/syncs/:baseId',
+  ncMetaAclMw(syncSourceList, 'syncSourceList')
+);
+router.post(
+  '/api/v1/db/meta/projects/:projectId/syncs/:baseId',
+  ncMetaAclMw(syncCreate, 'syncSourceCreate')
+);
 router.delete(
   '/api/v1/db/meta/syncs/:syncId',
   ncMetaAclMw(syncDelete, 'syncSourceDelete')

packages/nocodb/src/lib/meta/api/tableApis.ts

@@ -187,6 +187,7 @@ export async function tableCreate(req: Request<any, any, TableReqType>, res) {
 
   await Audit.insert({
     project_id: project.id,
+    base_id: base.id,
     op_type: AuditOperationTypes.TABLE,
     op_sub_type: AuditOperationSubTypes.CREATED,
     user: (req as any)?.user?.email,
@@ -357,6 +358,7 @@ export async function tableDelete(req: Request, res: Response) {
 
   await Audit.insert({
     project_id: project.id,
+    base_id: base.id,
     op_type: AuditOperationTypes.TABLE,
     op_sub_type: AuditOperationSubTypes.DELETED,
     user: (req as any)?.user?.email,
@@ -375,6 +377,11 @@ router.get(
   metaApiMetrics,
   ncMetaAclMw(tableList, 'tableList')
 );
+router.get(
+  '/api/v1/db/meta/projects/:projectId/:baseId/tables',
+  metaApiMetrics,
+  ncMetaAclMw(tableList, 'tableList')
+);
 router.post(
   '/api/v1/db/meta/projects/:projectId/tables',
   metaApiMetrics,

packages/nocodb/src/lib/migrations/XcMigrationSourcev2.ts

@@ -10,6 +10,7 @@ import * as nc_019_add_meta_in_meta_tables from './v2/nc_019_add_meta_in_meta_ta
 import * as nc_020_kanban_view from './v2/nc_020_kanban_view';
 import * as nc_021_add_fields_in_token from './v2/nc_021_add_fields_in_token';
 import * as nc_022_qr_code_column_type from './v2/nc_022_qr_code_column_type';
+import * as nc_023_add_base_id_in_sync_source from './v2/nc_023_add_base_id_in_sync_source';
 
 // Create a custom migration source class
 export default class XcMigrationSourcev2 {
@@ -31,6 +32,7 @@ export default class XcMigrationSourcev2 {
       'nc_020_kanban_view',
       'nc_021_add_fields_in_token',
       'nc_022_qr_code_column_type',
+      'nc_023_add_base_id_in_sync_source'
     ]);
   }
 
@@ -64,6 +66,8 @@ export default class XcMigrationSourcev2 {
         return nc_021_add_fields_in_token;
       case 'nc_022_qr_code_column_type':
         return nc_022_qr_code_column_type;
+      case 'nc_023_add_base_id_in_sync_source':
+        return nc_023_add_base_id_in_sync_source;
     }
   }
 }

packages/nocodb/src/lib/migrations/v2/nc_023_add_base_id_in_sync_source.ts

@@ -0,0 +1,39 @@
+import Knex from 'knex';
+import { MetaTable } from '../../utils/globals';
+
+const up = async (knex: Knex) => {
+  await knex.schema.alterTable(MetaTable.SYNC_SOURCE, (table) => {
+    table.string('base_id', 20);
+    table.foreign('base_id').references(`${MetaTable.BASES}.id`);
+  });
+};
+
+const down = async (knex) => {
+  await knex.schema.alterTable(MetaTable.SYNC_SOURCE, (table) => {
+    table.dropColumn('base_id');
+  });
+};
+
+export { up, down };
+
+/**
+ * @copyright Copyright (c) 2022, Xgene Cloud Ltd
+ *
+ * @author Mert Ersoy <mert@nocodb.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */

packages/nocodb/src/lib/models/SyncSource.ts

@@ -12,6 +12,7 @@ export default class SyncSource {
   deleted?: boolean;
   order?: number;
   project_id?: string;
+  base_id?: string;
   fk_user_id?: string;
 
   constructor(syncSource: Partial<SyncSource>) {
@@ -37,15 +38,14 @@ export default class SyncSource {
     return syncSource && new SyncSource(syncSource);
   }
 
-  static async list(projectId: string, ncMeta = Noco.ncMeta) {
+  static async list(projectId: string, baseId?: string, ncMeta = Noco.ncMeta) {
+    const condition = baseId ? { project_id: projectId, base_id: baseId } : { project_id: projectId };
     const syncSources = await ncMeta.metaList(
       null,
       null,
       MetaTable.SYNC_SOURCE,
       {
-        condition: {
+        condition,
-          project_id: projectId,
-        },
         orderBy: {
           created_at: 'asc',
         },
@@ -77,6 +77,7 @@ export default class SyncSource {
       type: syncSource?.type,
       details: syncSource?.details,
       project_id: syncSource?.project_id,
+      base_id: syncSource?.base_id,
       fk_user_id: syncSource?.fk_user_id,
     };
 
@@ -107,6 +108,7 @@ export default class SyncSource {
       'deleted',
       'order',
       'project_id',
+      'base_id',
     ]);
 
     if (updateObj.details && typeof updateObj.details === 'object') {

scripts/sdk/swagger.json

@@ -1743,6 +1743,55 @@
           "required": true
         }
       ],
+      "get": {
+        "summary": "",
+        "operationId": "table-list",
+        "responses": {
+          "200": {
+            "$ref": "#/components/responses/TableList"
+          }
+        },
+        "parameters": [
+          {
+            "schema": {
+              "type": "string"
+            },
+            "in": "header",
+            "name": "xc-auth"
+          },
+          {
+            "schema": {
+              "type": "number"
+            },
+            "in": "query",
+            "name": "page"
+          },
+          {
+            "schema": {
+              "type": "number"
+            },
+            "in": "query",
+            "name": "pageSize"
+          },
+          {
+            "schema": {
+              "type": "string"
+            },
+            "in": "query",
+            "name": "sort"
+          },
+          {
+            "schema": {
+              "type": "boolean"
+            },
+            "in": "query",
+            "name": "includeM2M"
+          }
+        ],
+        "tags": [
+          "Base"
+        ]
+      },
       "post": {
         "summary": "",
         "operationId": "table-create",