From 636e9babb987c0fb0874802ca6e5e51e16ca9670 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Sat, 11 Mar 2023 18:19:23 +0530 Subject: [PATCH] refactor: use service api's in airtable import Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../src/lib/controllers/sync/import.ctl.ts | 1 + .../nocodb/src/lib/services/column.svc.ts | 4 +- .../src/lib/services/sync/helpers/job.ts | 483 +++++++++++++----- .../sync/helpers/readAndProcessData.ts | 105 ++-- 4 files changed, 441 insertions(+), 152 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/sync/import.ctl.ts b/packages/nocodb/src/lib/controllers/sync/import.ctl.ts index d366e65b32..c497d80735 100644 --- a/packages/nocodb/src/lib/controllers/sync/import.ctl.ts +++ b/packages/nocodb/src/lib/controllers/sync/import.ctl.ts @@ -119,6 +119,7 @@ export default ( baseId: syncSource.base_id, authToken: token, baseURL, + user: user, }); }, 1000); diff --git a/packages/nocodb/src/lib/services/column.svc.ts b/packages/nocodb/src/lib/services/column.svc.ts index 3da6cba5ed..a24b3cfdd4 100644 --- a/packages/nocodb/src/lib/services/column.svc.ts +++ b/packages/nocodb/src/lib/services/column.svc.ts @@ -836,7 +836,7 @@ export async function columnSetAsPrimary(param: { columnId: string }) { } export async function columnAdd(param: { - req?: any; + req: any; tableId: string; column: ColumnReqType; }) { @@ -1111,7 +1111,7 @@ export async function columnAdd(param: { project_id: base.project_id, op_type: AuditOperationTypes.TABLE_COLUMN, op_sub_type: AuditOperationSubTypes.CREATED, - user: param?.req?.user?.email, + user: param?.req.user?.email, description: `created column ${colBody.column_name} with alias ${colBody.title} from table ${table.table_name}`, ip: param?.req.clientIp, }).then(() => {}); diff --git a/packages/nocodb/src/lib/services/sync/helpers/job.ts b/packages/nocodb/src/lib/services/sync/helpers/job.ts index be6a85ce63..f95b19132e 100644 --- a/packages/nocodb/src/lib/services/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/services/sync/helpers/job.ts @@ -2,8 +2,6 @@ import { promisify } from 'util'; import { UITypes } from 'nocodb-sdk'; // import * as sMap from './syncMap'; -import { Api } from 'nocodb-sdk'; - import Airtable from 'airtable'; import jsonfile from 'jsonfile'; import hash from 'object-hash'; @@ -12,10 +10,26 @@ import { T } from 'nc-help'; import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; import tinycolor from 'tinycolor2'; +import { + attachmentService, + columnService, + filterService, + formViewColumnService, + formViewService, + galleryViewService, + gridViewService, + projectService, + projectUserService, + sortService, + tableService, + viewColumnService, + viewService, +} from '../..'; import FetchAT from './fetchAT'; import { importData, importLTARData } from './readAndProcessData'; import EntityMap from './EntityMap'; +import type { UserType } from 'nocodb-sdk'; const writeJsonFileAsync = promisify(jsonfile.writeFile); @@ -74,6 +88,9 @@ export default async ( ) => { const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent'); await sMapEM.init(); + const userRole = syncDB.user.roles + .split(',') + .reduce((rolesObj, role) => ({ [role]: true, ...rolesObj }), {}); const sMap = { // static mapping records between aTblId && ncId @@ -121,7 +138,6 @@ export default async ( const enableErrorLogs = false; const generate_migrationStats = true; const debugMode = false; - let api: Api; let g_aTblSchema = []; let ncCreatedProjectSchema: any = {}; const ncLinkMappingTable: any[] = []; @@ -325,12 +341,24 @@ export default async ( // @ts-ignore async function nc_DumpTableSchema() { console.log('['); - const ncTblList = await api.base.tableList( - ncCreatedProjectSchema.id, - syncDB.baseId - ); + // const ncTblList = await api.base.tableList( + // ncCreatedProjectSchema.id, + // syncDB.baseId + // ); + + const ncTblList = { list: [] }; + ncTblList['list'] = await tableService.getAccessibleTables({ + projectId: ncCreatedProjectSchema.id, + baseId: syncDB.baseId, + roles: userRole, + }); + for (let i = 0; i < ncTblList.list.length; i++) { - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + // const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + const ncTbl = await tableService.getTableWithAccessibleViews({ + tableId: ncTblList.list[i].id, + user: syncDB.user, + }); console.log(JSON.stringify(ncTbl, null, 2)); console.log(','); } @@ -376,11 +404,18 @@ export default async ( projectId?: string; }) { // delete 'sample' project if already exists - const x = await api.project.list(); + // const x = await api.project.list(); + const x = { list: [] }; + x['list'] = await projectService.projectList({ + user: { id: syncDB.user.id, roles: syncDB.user.roles }, + }); const sampleProj = x.list.find((a) => a.title === projectName); if (sampleProj) { - await api.project.delete(sampleProj.id); + // await api.project.delete(sampleProj.id); + await projectService.projectSoftDelete({ + projectId: sampleProj.id, + }); } logDetailed('Init'); } @@ -630,11 +665,17 @@ export default async ( logDetailed(`NC API: base.tableCreate ${tables[idx].title}`); let _perfStart = recordPerfStart(); - const table: any = await api.base.tableCreate( - ncCreatedProjectSchema.id, - syncDB.baseId, - tables[idx] - ); + // const table: any = await api.base.tableCreate( + // ncCreatedProjectSchema.id, + // syncDB.baseId, + // tables[idx] + // ); + const table = await tableService.tableCreate({ + baseId: syncDB.baseId, + projectId: ncCreatedProjectSchema.id, + table: tables[idx], + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.create'); updateNcTblSchema(table); @@ -658,14 +699,23 @@ export default async ( // update default view name- to match it to airtable view name logDetailed(`NC API: dbView.list ${table.id}`); _perfStart = recordPerfStart(); - const view = await api.dbView.list(table.id); + // const view = await api.dbView.list(table.id); + const view = { list: [] }; + view['list'] = await viewService.viewList({ + tableId: table.id, + user: { roles: userRole }, + }); recordPerfStats(_perfStart, 'dbView.list'); const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid'); logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); _perfStart = recordPerfStart(); - await api.dbView.update(view.list[0].id, { - title: aTbl_grid.name, + // await api.dbView.update(view.list[0].id, { + // title: aTbl_grid.name, + // }); + await viewService.viewUpdate({ + viewId: view.list[0].id, + view: { title: aTbl_grid.name }, }); recordPerfStats(_perfStart, 'dbView.update'); @@ -730,7 +780,11 @@ export default async ( // check if already a column exists with this name? let _perfStart = recordPerfStart(); - const srcTbl: any = await api.dbTable.read(srcTableId); + // const srcTbl: any = await api.dbTable.read(srcTableId); + const srcTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: srcTableId, + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.read'); // create link @@ -747,16 +801,34 @@ export default async ( `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` ); _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.LinkToAnotherRecord, - title: ncName.title, - column_name: ncName.column_name, - parentId: srcTableId, - childId: childTableId, - type: 'mm', - // aTblLinkColumns[i].typeOptions.relationship === 'many' - // ? 'mm' - // : 'hm' + // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + // uidt: UITypes.LinkToAnotherRecord, + // title: ncName.title, + // column_name: ncName.column_name, + // parentId: srcTableId, + // childId: childTableId, + // type: 'mm', + // // aTblLinkColumns[i].typeOptions.relationship === 'many' + // // ? 'mm' + // // : 'hm' + // }); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.LinkToAnotherRecord, + title: ncName.title, + column_name: ncName.column_name, + parentId: srcTableId, + childId: childTableId, + type: 'mm', + // aTblLinkColumns[i].typeOptions.relationship === 'many' + // ? 'mm' + // : 'hm' + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -804,15 +876,25 @@ export default async ( ); let _perfStart = recordPerfStart(); - const childTblSchema: any = await api.dbTable.read( - ncLinkMappingTable[x].nc.childId - ); + // const childTblSchema: any = await api.dbTable.read( + // ncLinkMappingTable[x].nc.childId + // ); + const childTblSchema: any = + await tableService.getTableWithAccessibleViews({ + tableId: ncLinkMappingTable[x].nc.childId, + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.read'); _perfStart = recordPerfStart(); - const parentTblSchema: any = await api.dbTable.read( - ncLinkMappingTable[x].nc.parentId - ); + // const parentTblSchema: any = await api.dbTable.read( + // ncLinkMappingTable[x].nc.parentId + // ); + const parentTblSchema: any = + await tableService.getTableWithAccessibleViews({ + tableId: ncLinkMappingTable[x].nc.parentId, + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.read'); // fix me @@ -891,14 +973,22 @@ export default async ( `NC API: dbTableColumn.update rename symmetric column ${ncName.title}` ); _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.update( - childLinkColumn.id, - { + // const ncTbl: any = await api.dbTableColumn.update( + // childLinkColumn.id, + // { + // ...childLinkColumn, + // title: ncName.title, + // column_name: ncName.column_name, + // } + // ); + const ncTbl: any = await columnService.columnUpdate({ + columnId: childLinkColumn.id, + column: { ...childLinkColumn, title: ncName.title, column_name: ncName.column_name, - } - ); + }, + }); recordPerfStats(_perfStart, 'dbTableColumn.update'); updateNcTblSchema(ncTbl); @@ -980,12 +1070,26 @@ export default async ( logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, + // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + // uidt: UITypes.Lookup, + // title: ncName.title, + // column_name: ncName.column_name, + // fk_relation_column_id: ncRelationColumnId, + // fk_lookup_column_id: ncLookupColumnId, + // }); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1060,12 +1164,26 @@ export default async ( logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, + // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + // uidt: UITypes.Lookup, + // title: ncName.title, + // column_name: ncName.column_name, + // fk_relation_column_id: ncRelationColumnId, + // fk_lookup_column_id: ncLookupColumnId, + // }); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1203,13 +1321,28 @@ export default async ( logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Rollup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_rollup_column_id: ncRollupColumnId, - rollup_function: ncRollupFn, + // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + // uidt: UITypes.Rollup, + // title: ncName.title, + // column_name: ncName.column_name, + // fk_relation_column_id: ncRelationColumnId, + // fk_rollup_column_id: ncRollupColumnId, + // rollup_function: ncRollupFn, + // }); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Rollup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_rollup_column_id: ncRollupColumnId, + rollup_function: ncRollupFn, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1261,12 +1394,26 @@ export default async ( logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, + // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + // uidt: UITypes.Lookup, + // title: ncName.title, + // column_name: ncName.column_name, + // fk_relation_column_id: ncRelationColumnId, + // fk_lookup_column_id: ncLookupColumnId, + // }); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1302,7 +1449,8 @@ export default async ( if (ncColId) { logDetailed(`NC API: dbTableColumn.primaryColumnSet`); const _perfStart = recordPerfStart(); - await api.dbTableColumn.primaryColumnSet(ncColId); + // await api.dbTableColumn.primaryColumnSet(ncColId); + await columnService.columnSetAsPrimary({ columnId: ncColId }); recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet'); // update schema @@ -1319,13 +1467,19 @@ export default async ( const _perfStart = recordPerfStart(); if (viewType === 'form') { - viewDetails = (await api.dbView.formRead(viewId)).columns; + // viewDetails = (await api.dbView.formRead(viewId)).columns; + viewDetails = (await formViewService.formViewGet({ formViewId: viewId })) + .columns; recordPerfStats(_perfStart, 'dbView.formRead'); } else if (viewType === 'gallery') { - viewDetails = (await api.dbView.galleryRead(viewId)).columns; + // viewDetails = (await api.dbView.galleryRead(viewId)).columns; + viewDetails = ( + await galleryViewService.galleryViewGet({ galleryViewId: viewId }) + ).columns; recordPerfStats(_perfStart, 'dbView.galleryRead'); } else { - viewDetails = await api.dbView.gridColumnsList(viewId); + // viewDetails = await api.dbView.gridColumnsList(viewId); + viewDetails = await viewColumnService.columnList({ viewId: viewId }); recordPerfStats(_perfStart, 'dbView.gridColumnsList'); } @@ -1453,17 +1607,26 @@ export default async ( ?.map((a) => a.filename?.split('?')?.[0]) .join(', ')}` ); - tempArr = await api.storage.uploadByUrl( - { - path: `noco/${sDB.projectName}/${table.title}/${key}`, - }, - value?.map((attachment) => ({ + // tempArr = await api.storage.uploadByUrl( + // { + // path: `noco/${sDB.projectName}/${table.title}/${key}`, + // }, + // value?.map((attachment) => ({ + // fileName: attachment.filename?.split('?')?.[0], + // url: attachment.url, + // size: attachment.size, + // mimetype: attachment.type, + // })) + // ); + tempArr = await attachmentService.uploadViaURL({ + path: `noco/${sDB.projectName}/${table.title}/${key}`, + urls: value?.map((attachment) => ({ fileName: attachment.filename?.split('?')?.[0], url: attachment.url, size: attachment.size, mimetype: attachment.type, - })) - ); + })), + }); } catch (e) { console.log(e); } @@ -1543,9 +1706,15 @@ export default async ( // create empty project (XC-DB) logDetailed(`Create Project: ${projName}`); const _perfStart = recordPerfStart(); - ncCreatedProjectSchema = await api.project.create({ - title: projName, + // ncCreatedProjectSchema = await api.project.create({ + // title: projName, + // }); + + ncCreatedProjectSchema = await projectService.projectCreate({ + project: { title: projName }, + user: { id: syncDB.user.id }, }); + recordPerfStats(_perfStart, 'project.create'); } @@ -1553,7 +1722,10 @@ export default async ( // create empty project (XC-DB) logDetailed(`Getting project meta: ${projId}`); const _perfStart = recordPerfStart(); - ncCreatedProjectSchema = await api.project.read(projId); + // ncCreatedProjectSchema = await api.project.read(projId); + ncCreatedProjectSchema = await projectService.getProjectWithInfo({ + projectId: projId, + }); recordPerfStats(_perfStart, 'project.read'); } @@ -1585,7 +1757,13 @@ export default async ( logDetailed(`NC API dbView.galleryCreate :: ${viewName}`); const _perfStart = recordPerfStart(); - await api.dbView.galleryCreate(tblId, { title: viewName }); + // await api.dbView.galleryCreate(tblId, { title: viewName }); + await galleryViewService.galleryViewCreate({ + tableId: tblId, + gallery: { + title: viewName, + }, + }); recordPerfStats(_perfStart, 'dbView.galleryCreate'); await updateNcTblSchemaById(tblId); @@ -1646,7 +1824,11 @@ export default async ( logDetailed(`NC API dbView.formCreate :: ${viewName}`); const _perfStart = recordPerfStart(); - const f = await api.dbView.formCreate(tblId, formData); + // const f = await api.dbView.formCreate(tblId, formData); + const f = await formViewService.formViewCreate({ + tableId: tblId, + body: formData, + }); recordPerfStats(_perfStart, 'dbView.formCreate'); logDetailed( @@ -1689,7 +1871,12 @@ export default async ( (x) => x.id === gridViews[i].id )?.name; const _perfStart = recordPerfStart(); - const viewList: any = await api.dbView.list(tblId); + // const viewList: any = await api.dbView.list(tblId); + const viewList = { list: [] }; + viewList['list'] = await viewService.viewList({ + tableId: tblId, + user: { roles: userRole }, + }); recordPerfStats(_perfStart, 'dbView.list'); let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id; @@ -1704,8 +1891,14 @@ export default async ( if (i > 0) { logDetailed(`NC API dbView.gridCreate :: ${viewName}`); const _perfStart = recordPerfStart(); - const viewCreated = await api.dbView.gridCreate(tblId, { - title: viewName, + // const viewCreated = await api.dbView.gridCreate(tblId, { + // title: viewName, + // }); + const viewCreated = await gridViewService.gridViewCreate({ + tableId: tblId, + grid: { + title: viewName, + }, }); recordPerfStats(_perfStart, 'dbView.gridCreate'); @@ -1773,10 +1966,19 @@ export default async ( ); const _perfStart = recordPerfStart(); insertJobs.push( - api.auth - .projectUserAdd(ncCreatedProjectSchema.id, { - email: value.email, - roles: userRoles[value.permissionLevel], + // api.auth + // .projectUserAdd(ncCreatedProjectSchema.id, { + // email: value.email, + // roles: userRoles[value.permissionLevel], + // }) + projectUserService + .userInvite({ + projectId: ncCreatedProjectSchema.id, + projectUser: { + email: value.email, + roles: userRoles[value.permissionLevel], + }, + req: { user: syncDB.user, clientIp: '' }, }) .catch((e) => e.response?.data?.msg @@ -1803,7 +2005,11 @@ export default async ( async function updateNcTblSchemaById(tblId) { const _perfStart = recordPerfStart(); - const ncTbl = await api.dbTable.read(tblId); + // const ncTbl = await api.dbTable.read(tblId); + const ncTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: tblId, + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.read'); updateNcTblSchema(ncTbl); @@ -2060,8 +2266,12 @@ export default async ( // insert filters for (let i = 0; i < ncFilters.length; i++) { const _perfStart = recordPerfStart(); - await api.dbTableFilter.create(viewId, { - ...ncFilters[i], + // await api.dbTableFilter.create(viewId, { + // ...ncFilters[i], + // }); + await filterService.filterCreate({ + viewId: viewId, + filter: ncFilters[i], }); recordPerfStats(_perfStart, 'dbTableFilter.create'); @@ -2076,9 +2286,16 @@ export default async ( if (columnId) { const _perfStart = recordPerfStart(); - await api.dbTableSort.create(viewId, { - fk_column_id: columnId, - direction: s.sortSet[i].ascending ? 'asc' : 'desc', + // await api.dbTableSort.create(viewId, { + // fk_column_id: columnId, + // direction: s.sortSet[i].ascending ? 'asc' : 'desc', + // }); + await sortService.sortCreate({ + viewId: viewId, + sort: { + fk_column_id: columnId, + direction: s.sortSet[i].ascending ? 'asc' : 'desc', + }, }); recordPerfStats(_perfStart, 'dbTableSort.create'); } @@ -2100,13 +2317,21 @@ export default async ( const _perfStart = recordPerfStart(); if (viewType === 'form') { - viewDetails = (await api.dbView.formRead(viewId)).columns; + // viewDetails = (await api.dbView.formRead(viewId)).columns; + viewDetails = (await formViewService.formViewGet({ formViewId: viewId })) + .columns; recordPerfStats(_perfStart, 'dbView.formRead'); } else if (viewType === 'gallery') { - viewDetails = (await api.dbView.galleryRead(viewId)).columns; + // viewDetails = (await api.dbView.galleryRead(viewId)).columns; + viewDetails = ( + await galleryViewService.galleryViewGet({ + galleryViewId: viewId, + }) + ).columns; recordPerfStats(_perfStart, 'dbView.galleryRead'); } else { - viewDetails = await api.dbView.gridColumnsList(viewId); + // viewDetails = await api.dbView.gridColumnsList(viewId); + viewDetails = await viewColumnService.columnList({ viewId: viewId }); recordPerfStats(_perfStart, 'dbView.gridColumnsList'); } @@ -2127,9 +2352,17 @@ export default async ( // first two positions held by record id & record hash const _perfStart = recordPerfStart(); - await api.dbViewColumn.update(viewId, ncViewColumnId, { - show: false, - order: j + 1 + c.length, + // await api.dbViewColumn.update(viewId, ncViewColumnId, { + // show: false, + // order: j + 1 + c.length, + // }); + await viewColumnService.columnUpdate({ + viewId: viewId, + columnId: ncViewColumnId, + column: { + show: false, + order: j + 1 + c.length, + }, }); recordPerfStats(_perfStart, 'dbViewColumn.update'); } @@ -2154,12 +2387,21 @@ export default async ( if (x?.required) formData[`required`] = x.required; if (x?.description) formData[`description`] = x.description; const _perfStart = recordPerfStart(); - await api.dbView.formColumnUpdate(ncViewColumnId, formData); + // await api.dbView.formColumnUpdate(ncViewColumnId, formData); + await formViewColumnService.columnUpdate({ + formViewColumnId: ncViewColumnId, + formViewColumn: formData, + }); recordPerfStats(_perfStart, 'dbView.formColumnUpdate'); } } const _perfStart = recordPerfStart(); - await api.dbViewColumn.update(viewId, ncViewColumnId, configData); + // await api.dbViewColumn.update(viewId, ncViewColumnId, configData); + await viewColumnService.columnUpdate({ + viewId: viewId, + columnId: ncViewColumnId, + column: configData, + }); recordPerfStats(_perfStart, 'dbViewColumn.update'); } } @@ -2168,13 +2410,6 @@ export default async ( let recordCnt = 0; try { logBasic('SDK initialized'); - api = new Api({ - baseURL: syncDB.baseURL, - headers: { - 'xc-auth': syncDB.authToken, - }, - }); - logDetailed('Project initialization started'); // delete project if already exists if (debugMode) await init(syncDB); @@ -2254,10 +2489,16 @@ export default async ( try { // await nc_DumpTableSchema(); const _perfStart = recordPerfStart(); - const ncTblList = await api.base.tableList( - ncCreatedProjectSchema.id, - syncDB.baseId - ); + // const ncTblList = await api.base.tableList( + // ncCreatedProjectSchema.id, + // syncDB.baseId + // ); + const ncTblList = { list: [] }; + ncTblList['list'] = await tableService.getAccessibleTables({ + projectId: ncCreatedProjectSchema.id, + baseId: syncDB.baseId, + roles: userRole, + }); recordPerfStats(_perfStart, 'base.tableList'); logBasic('Reading Records...'); @@ -2273,7 +2514,11 @@ export default async ( continue; const _perfStart = recordPerfStart(); - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + // const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + const ncTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: ncTblList.list[i].id, + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.read'); recordCnt = 0; @@ -2283,7 +2528,6 @@ export default async ( projectName: syncDB.projectName, table: ncTbl, base, - api, logBasic, nocoBaseDataProcessing_v2, sDB: syncDB, @@ -2303,12 +2547,15 @@ export default async ( ) continue; - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + // const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + const ncTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: ncTblList.list[i].id, + user: syncDB.user, + }); rtc.data.nestedLinks += await importLTARData({ table: ncTbl, projectName: syncDB.projectName, - api, base, fields: null, //Object.values(tblLinkGroup).flat(), logBasic, @@ -2317,6 +2564,7 @@ export default async ( records: recordsMap[ncTbl.id], atNcAliasRef, ncLinkMappingTable, + syncDB, }); } @@ -2417,6 +2665,7 @@ export interface AirtableSyncConfig { baseId?: string; apiKey: string; shareId: string; + user: UserType; options: { syncViews: boolean; syncData: boolean; diff --git a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts index faf06482a0..7dcd31d813 100644 --- a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts @@ -1,7 +1,8 @@ import { RelationTypes, UITypes } from 'nocodb-sdk'; +import { bulkDataService, tableService } from '../..'; import EntityMap from './EntityMap'; import type { AirtableBase } from 'airtable/lib/airtable_base'; -import type { Api, TableType } from 'nocodb-sdk'; +import type { TableType } from 'nocodb-sdk'; const BULK_DATA_BATCH_SIZE = 500; const ASSOC_BULK_DATA_BATCH_SIZE = 1000; @@ -70,7 +71,6 @@ export async function importData({ projectName, table, base, - api, nocoBaseDataProcessing_v2, sDB, logDetailed = (_str) => {}, @@ -82,7 +82,6 @@ export async function importData({ base: AirtableBase; logBasic: (string) => void; logDetailed: (string) => void; - api: Api; nocoBaseDataProcessing_v2; sDB; }): Promise { @@ -116,12 +115,20 @@ export async function importData({ if (tempData.length >= BULK_DATA_BATCH_SIZE) { let insertArray = tempData.splice(0, tempData.length); - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - table.id, - insertArray - ); + // await api.dbTableRow.bulkCreate( + // 'nc', + // projectName, + // table.id, + // insertArray + // ); + + await bulkDataService.bulkDataInsert({ + projectName: projectName, + tableName: table.title, + body: insertArray, + cookie: {}, + }); + logBasic( `:: Importing '${ table.title @@ -142,12 +149,20 @@ export async function importData({ readable.on('end', async () => { await Promise.all(promises); if (tempData.length > 0) { - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - table.id, - tempData - ); + // await api.dbTableRow.bulkCreate( + // 'nc', + // projectName, + // table.id, + // tempData + // ); + + await bulkDataService.bulkDataInsert({ + projectName: projectName, + tableName: table.title, + body: tempData, + cookie: {}, + }); + logBasic( `:: Importing '${ table.title @@ -174,7 +189,6 @@ export async function importLTARData({ table, fields, base, - api, projectName, insertedAssocRef = {}, logDetailed = (_str) => {}, @@ -182,6 +196,7 @@ export async function importLTARData({ records, atNcAliasRef, ncLinkMappingTable, + syncDB, }: { projectName: string; table: { title?: string; id?: string }; @@ -189,7 +204,6 @@ export async function importLTARData({ base: AirtableBase; logDetailed: (string) => void; logBasic: (string) => void; - api: Api; insertedAssocRef: { [assocTableId: string]: boolean }; records?: EntityMap; atNcAliasRef: { @@ -198,6 +212,7 @@ export async function importLTARData({ }; }; ncLinkMappingTable: Record>[]; + syncDB; }) { const assocTableMetas: Array<{ modelMeta: { id?: string; title?: string }; @@ -215,7 +230,11 @@ export async function importLTARData({ logBasic, })); - const modelMeta: any = await api.dbTable.read(table.id); + // const modelMeta: any = await api.dbTable.read(table.id); + const modelMeta: any = await tableService.getTableWithAccessibleViews({ + tableId: table.id, + user: syncDB.user, + }); for (const colMeta of modelMeta.columns) { // skip columns which are not LTAR and Many to many @@ -235,9 +254,15 @@ export async function importLTARData({ // mark as inserted insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; - const assocModelMeta: TableType = (await api.dbTable.read( - colMeta.colOptions.fk_mm_model_id - )) as any; + // const assocModelMeta: TableType = (await api.dbTable.read( + // colMeta.colOptions.fk_mm_model_id + // )) as any; + + const assocModelMeta: TableType = + (await tableService.getTableWithAccessibleViews({ + tableId: colMeta.colOptions.fk_mm_model_id, + user: syncDB.user, + })) as any; // extract associative table and columns meta assocTableMetas.push({ @@ -291,12 +316,19 @@ export async function importLTARData({ )}` ); - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - assocMeta.modelMeta.id, - insertArray - ); + // await api.dbTableRow.bulkCreate( + // 'nc', + // projectName, + // assocMeta.modelMeta.id, + // insertArray + // ); + + await bulkDataService.bulkDataInsert({ + projectName: projectName, + tableName: table.title, + body: insertArray, + cookie: {}, + }); importedCount += insertArray.length; insertArray = []; @@ -319,12 +351,19 @@ export async function importLTARData({ )}` ); - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - assocMeta.modelMeta.id, - assocTableData - ); + // await api.dbTableRow.bulkCreate( + // 'nc', + // projectName, + // assocMeta.modelMeta.id, + // assocTableData + // ); + + await bulkDataService.bulkDataInsert({ + projectName: projectName, + tableName: table.title, + body: assocTableData, + cookie: {}, + }); importedCount += assocTableData.length; assocTableData = [];