From 636e9babb987c0fb0874802ca6e5e51e16ca9670 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Sat, 11 Mar 2023 18:19:23 +0530 Subject: [PATCH 1/7] refactor: use service api's in airtable import Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../src/lib/controllers/sync/import.ctl.ts | 1 + .../nocodb/src/lib/services/column.svc.ts | 4 +- .../src/lib/services/sync/helpers/job.ts | 483 +++++++++++++----- .../sync/helpers/readAndProcessData.ts | 105 ++-- 4 files changed, 441 insertions(+), 152 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/sync/import.ctl.ts b/packages/nocodb/src/lib/controllers/sync/import.ctl.ts index d366e65b32..c497d80735 100644 --- a/packages/nocodb/src/lib/controllers/sync/import.ctl.ts +++ b/packages/nocodb/src/lib/controllers/sync/import.ctl.ts @@ -119,6 +119,7 @@ export default ( baseId: syncSource.base_id, authToken: token, baseURL, + user: user, }); }, 1000); diff --git a/packages/nocodb/src/lib/services/column.svc.ts b/packages/nocodb/src/lib/services/column.svc.ts index 3da6cba5ed..a24b3cfdd4 100644 --- a/packages/nocodb/src/lib/services/column.svc.ts +++ b/packages/nocodb/src/lib/services/column.svc.ts @@ -836,7 +836,7 @@ export async function columnSetAsPrimary(param: { columnId: string }) { } export async function columnAdd(param: { - req?: any; + req: any; tableId: string; column: ColumnReqType; }) { @@ -1111,7 +1111,7 @@ export async function columnAdd(param: { project_id: base.project_id, op_type: AuditOperationTypes.TABLE_COLUMN, op_sub_type: AuditOperationSubTypes.CREATED, - user: param?.req?.user?.email, + user: param?.req.user?.email, description: `created column ${colBody.column_name} with alias ${colBody.title} from table ${table.table_name}`, ip: param?.req.clientIp, }).then(() => {}); diff --git a/packages/nocodb/src/lib/services/sync/helpers/job.ts b/packages/nocodb/src/lib/services/sync/helpers/job.ts index be6a85ce63..f95b19132e 100644 --- a/packages/nocodb/src/lib/services/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/services/sync/helpers/job.ts @@ -2,8 +2,6 @@ import { promisify } from 'util'; import { UITypes } from 'nocodb-sdk'; // import * as sMap from './syncMap'; -import { Api } from 'nocodb-sdk'; - import Airtable from 'airtable'; import jsonfile from 'jsonfile'; import hash from 'object-hash'; @@ -12,10 +10,26 @@ import { T } from 'nc-help'; import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; import tinycolor from 'tinycolor2'; +import { + attachmentService, + columnService, + filterService, + formViewColumnService, + formViewService, + galleryViewService, + gridViewService, + projectService, + projectUserService, + sortService, + tableService, + viewColumnService, + viewService, +} from '../..'; import FetchAT from './fetchAT'; import { importData, importLTARData } from './readAndProcessData'; import EntityMap from './EntityMap'; +import type { UserType } from 'nocodb-sdk'; const writeJsonFileAsync = promisify(jsonfile.writeFile); @@ -74,6 +88,9 @@ export default async ( ) => { const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent'); await sMapEM.init(); + const userRole = syncDB.user.roles + .split(',') + .reduce((rolesObj, role) => ({ [role]: true, ...rolesObj }), {}); const sMap = { // static mapping records between aTblId && ncId @@ -121,7 +138,6 @@ export default async ( const enableErrorLogs = false; const generate_migrationStats = true; const debugMode = false; - let api: Api; let g_aTblSchema = []; let ncCreatedProjectSchema: any = {}; const ncLinkMappingTable: any[] = []; @@ -325,12 +341,24 @@ export default async ( // @ts-ignore async function nc_DumpTableSchema() { console.log('['); - const ncTblList = await api.base.tableList( - ncCreatedProjectSchema.id, - syncDB.baseId - ); + // const ncTblList = await api.base.tableList( + // ncCreatedProjectSchema.id, + // syncDB.baseId + // ); + + const ncTblList = { list: [] }; + ncTblList['list'] = await tableService.getAccessibleTables({ + projectId: ncCreatedProjectSchema.id, + baseId: syncDB.baseId, + roles: userRole, + }); + for (let i = 0; i < ncTblList.list.length; i++) { - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + // const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + const ncTbl = await tableService.getTableWithAccessibleViews({ + tableId: ncTblList.list[i].id, + user: syncDB.user, + }); console.log(JSON.stringify(ncTbl, null, 2)); console.log(','); } @@ -376,11 +404,18 @@ export default async ( projectId?: string; }) { // delete 'sample' project if already exists - const x = await api.project.list(); + // const x = await api.project.list(); + const x = { list: [] }; + x['list'] = await projectService.projectList({ + user: { id: syncDB.user.id, roles: syncDB.user.roles }, + }); const sampleProj = x.list.find((a) => a.title === projectName); if (sampleProj) { - await api.project.delete(sampleProj.id); + // await api.project.delete(sampleProj.id); + await projectService.projectSoftDelete({ + projectId: sampleProj.id, + }); } logDetailed('Init'); } @@ -630,11 +665,17 @@ export default async ( logDetailed(`NC API: base.tableCreate ${tables[idx].title}`); let _perfStart = recordPerfStart(); - const table: any = await api.base.tableCreate( - ncCreatedProjectSchema.id, - syncDB.baseId, - tables[idx] - ); + // const table: any = await api.base.tableCreate( + // ncCreatedProjectSchema.id, + // syncDB.baseId, + // tables[idx] + // ); + const table = await tableService.tableCreate({ + baseId: syncDB.baseId, + projectId: ncCreatedProjectSchema.id, + table: tables[idx], + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.create'); updateNcTblSchema(table); @@ -658,14 +699,23 @@ export default async ( // update default view name- to match it to airtable view name logDetailed(`NC API: dbView.list ${table.id}`); _perfStart = recordPerfStart(); - const view = await api.dbView.list(table.id); + // const view = await api.dbView.list(table.id); + const view = { list: [] }; + view['list'] = await viewService.viewList({ + tableId: table.id, + user: { roles: userRole }, + }); recordPerfStats(_perfStart, 'dbView.list'); const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid'); logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); _perfStart = recordPerfStart(); - await api.dbView.update(view.list[0].id, { - title: aTbl_grid.name, + // await api.dbView.update(view.list[0].id, { + // title: aTbl_grid.name, + // }); + await viewService.viewUpdate({ + viewId: view.list[0].id, + view: { title: aTbl_grid.name }, }); recordPerfStats(_perfStart, 'dbView.update'); @@ -730,7 +780,11 @@ export default async ( // check if already a column exists with this name? let _perfStart = recordPerfStart(); - const srcTbl: any = await api.dbTable.read(srcTableId); + // const srcTbl: any = await api.dbTable.read(srcTableId); + const srcTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: srcTableId, + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.read'); // create link @@ -747,16 +801,34 @@ export default async ( `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` ); _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.LinkToAnotherRecord, - title: ncName.title, - column_name: ncName.column_name, - parentId: srcTableId, - childId: childTableId, - type: 'mm', - // aTblLinkColumns[i].typeOptions.relationship === 'many' - // ? 'mm' - // : 'hm' + // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + // uidt: UITypes.LinkToAnotherRecord, + // title: ncName.title, + // column_name: ncName.column_name, + // parentId: srcTableId, + // childId: childTableId, + // type: 'mm', + // // aTblLinkColumns[i].typeOptions.relationship === 'many' + // // ? 'mm' + // // : 'hm' + // }); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.LinkToAnotherRecord, + title: ncName.title, + column_name: ncName.column_name, + parentId: srcTableId, + childId: childTableId, + type: 'mm', + // aTblLinkColumns[i].typeOptions.relationship === 'many' + // ? 'mm' + // : 'hm' + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -804,15 +876,25 @@ export default async ( ); let _perfStart = recordPerfStart(); - const childTblSchema: any = await api.dbTable.read( - ncLinkMappingTable[x].nc.childId - ); + // const childTblSchema: any = await api.dbTable.read( + // ncLinkMappingTable[x].nc.childId + // ); + const childTblSchema: any = + await tableService.getTableWithAccessibleViews({ + tableId: ncLinkMappingTable[x].nc.childId, + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.read'); _perfStart = recordPerfStart(); - const parentTblSchema: any = await api.dbTable.read( - ncLinkMappingTable[x].nc.parentId - ); + // const parentTblSchema: any = await api.dbTable.read( + // ncLinkMappingTable[x].nc.parentId + // ); + const parentTblSchema: any = + await tableService.getTableWithAccessibleViews({ + tableId: ncLinkMappingTable[x].nc.parentId, + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.read'); // fix me @@ -891,14 +973,22 @@ export default async ( `NC API: dbTableColumn.update rename symmetric column ${ncName.title}` ); _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.update( - childLinkColumn.id, - { + // const ncTbl: any = await api.dbTableColumn.update( + // childLinkColumn.id, + // { + // ...childLinkColumn, + // title: ncName.title, + // column_name: ncName.column_name, + // } + // ); + const ncTbl: any = await columnService.columnUpdate({ + columnId: childLinkColumn.id, + column: { ...childLinkColumn, title: ncName.title, column_name: ncName.column_name, - } - ); + }, + }); recordPerfStats(_perfStart, 'dbTableColumn.update'); updateNcTblSchema(ncTbl); @@ -980,12 +1070,26 @@ export default async ( logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, + // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + // uidt: UITypes.Lookup, + // title: ncName.title, + // column_name: ncName.column_name, + // fk_relation_column_id: ncRelationColumnId, + // fk_lookup_column_id: ncLookupColumnId, + // }); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1060,12 +1164,26 @@ export default async ( logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, + // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + // uidt: UITypes.Lookup, + // title: ncName.title, + // column_name: ncName.column_name, + // fk_relation_column_id: ncRelationColumnId, + // fk_lookup_column_id: ncLookupColumnId, + // }); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1203,13 +1321,28 @@ export default async ( logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Rollup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_rollup_column_id: ncRollupColumnId, - rollup_function: ncRollupFn, + // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + // uidt: UITypes.Rollup, + // title: ncName.title, + // column_name: ncName.column_name, + // fk_relation_column_id: ncRelationColumnId, + // fk_rollup_column_id: ncRollupColumnId, + // rollup_function: ncRollupFn, + // }); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Rollup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_rollup_column_id: ncRollupColumnId, + rollup_function: ncRollupFn, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1261,12 +1394,26 @@ export default async ( logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, + // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { + // uidt: UITypes.Lookup, + // title: ncName.title, + // column_name: ncName.column_name, + // fk_relation_column_id: ncRelationColumnId, + // fk_lookup_column_id: ncLookupColumnId, + // }); + const ncTbl: any = await columnService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Lookup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_lookup_column_id: ncLookupColumnId, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1302,7 +1449,8 @@ export default async ( if (ncColId) { logDetailed(`NC API: dbTableColumn.primaryColumnSet`); const _perfStart = recordPerfStart(); - await api.dbTableColumn.primaryColumnSet(ncColId); + // await api.dbTableColumn.primaryColumnSet(ncColId); + await columnService.columnSetAsPrimary({ columnId: ncColId }); recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet'); // update schema @@ -1319,13 +1467,19 @@ export default async ( const _perfStart = recordPerfStart(); if (viewType === 'form') { - viewDetails = (await api.dbView.formRead(viewId)).columns; + // viewDetails = (await api.dbView.formRead(viewId)).columns; + viewDetails = (await formViewService.formViewGet({ formViewId: viewId })) + .columns; recordPerfStats(_perfStart, 'dbView.formRead'); } else if (viewType === 'gallery') { - viewDetails = (await api.dbView.galleryRead(viewId)).columns; + // viewDetails = (await api.dbView.galleryRead(viewId)).columns; + viewDetails = ( + await galleryViewService.galleryViewGet({ galleryViewId: viewId }) + ).columns; recordPerfStats(_perfStart, 'dbView.galleryRead'); } else { - viewDetails = await api.dbView.gridColumnsList(viewId); + // viewDetails = await api.dbView.gridColumnsList(viewId); + viewDetails = await viewColumnService.columnList({ viewId: viewId }); recordPerfStats(_perfStart, 'dbView.gridColumnsList'); } @@ -1453,17 +1607,26 @@ export default async ( ?.map((a) => a.filename?.split('?')?.[0]) .join(', ')}` ); - tempArr = await api.storage.uploadByUrl( - { - path: `noco/${sDB.projectName}/${table.title}/${key}`, - }, - value?.map((attachment) => ({ + // tempArr = await api.storage.uploadByUrl( + // { + // path: `noco/${sDB.projectName}/${table.title}/${key}`, + // }, + // value?.map((attachment) => ({ + // fileName: attachment.filename?.split('?')?.[0], + // url: attachment.url, + // size: attachment.size, + // mimetype: attachment.type, + // })) + // ); + tempArr = await attachmentService.uploadViaURL({ + path: `noco/${sDB.projectName}/${table.title}/${key}`, + urls: value?.map((attachment) => ({ fileName: attachment.filename?.split('?')?.[0], url: attachment.url, size: attachment.size, mimetype: attachment.type, - })) - ); + })), + }); } catch (e) { console.log(e); } @@ -1543,9 +1706,15 @@ export default async ( // create empty project (XC-DB) logDetailed(`Create Project: ${projName}`); const _perfStart = recordPerfStart(); - ncCreatedProjectSchema = await api.project.create({ - title: projName, + // ncCreatedProjectSchema = await api.project.create({ + // title: projName, + // }); + + ncCreatedProjectSchema = await projectService.projectCreate({ + project: { title: projName }, + user: { id: syncDB.user.id }, }); + recordPerfStats(_perfStart, 'project.create'); } @@ -1553,7 +1722,10 @@ export default async ( // create empty project (XC-DB) logDetailed(`Getting project meta: ${projId}`); const _perfStart = recordPerfStart(); - ncCreatedProjectSchema = await api.project.read(projId); + // ncCreatedProjectSchema = await api.project.read(projId); + ncCreatedProjectSchema = await projectService.getProjectWithInfo({ + projectId: projId, + }); recordPerfStats(_perfStart, 'project.read'); } @@ -1585,7 +1757,13 @@ export default async ( logDetailed(`NC API dbView.galleryCreate :: ${viewName}`); const _perfStart = recordPerfStart(); - await api.dbView.galleryCreate(tblId, { title: viewName }); + // await api.dbView.galleryCreate(tblId, { title: viewName }); + await galleryViewService.galleryViewCreate({ + tableId: tblId, + gallery: { + title: viewName, + }, + }); recordPerfStats(_perfStart, 'dbView.galleryCreate'); await updateNcTblSchemaById(tblId); @@ -1646,7 +1824,11 @@ export default async ( logDetailed(`NC API dbView.formCreate :: ${viewName}`); const _perfStart = recordPerfStart(); - const f = await api.dbView.formCreate(tblId, formData); + // const f = await api.dbView.formCreate(tblId, formData); + const f = await formViewService.formViewCreate({ + tableId: tblId, + body: formData, + }); recordPerfStats(_perfStart, 'dbView.formCreate'); logDetailed( @@ -1689,7 +1871,12 @@ export default async ( (x) => x.id === gridViews[i].id )?.name; const _perfStart = recordPerfStart(); - const viewList: any = await api.dbView.list(tblId); + // const viewList: any = await api.dbView.list(tblId); + const viewList = { list: [] }; + viewList['list'] = await viewService.viewList({ + tableId: tblId, + user: { roles: userRole }, + }); recordPerfStats(_perfStart, 'dbView.list'); let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id; @@ -1704,8 +1891,14 @@ export default async ( if (i > 0) { logDetailed(`NC API dbView.gridCreate :: ${viewName}`); const _perfStart = recordPerfStart(); - const viewCreated = await api.dbView.gridCreate(tblId, { - title: viewName, + // const viewCreated = await api.dbView.gridCreate(tblId, { + // title: viewName, + // }); + const viewCreated = await gridViewService.gridViewCreate({ + tableId: tblId, + grid: { + title: viewName, + }, }); recordPerfStats(_perfStart, 'dbView.gridCreate'); @@ -1773,10 +1966,19 @@ export default async ( ); const _perfStart = recordPerfStart(); insertJobs.push( - api.auth - .projectUserAdd(ncCreatedProjectSchema.id, { - email: value.email, - roles: userRoles[value.permissionLevel], + // api.auth + // .projectUserAdd(ncCreatedProjectSchema.id, { + // email: value.email, + // roles: userRoles[value.permissionLevel], + // }) + projectUserService + .userInvite({ + projectId: ncCreatedProjectSchema.id, + projectUser: { + email: value.email, + roles: userRoles[value.permissionLevel], + }, + req: { user: syncDB.user, clientIp: '' }, }) .catch((e) => e.response?.data?.msg @@ -1803,7 +2005,11 @@ export default async ( async function updateNcTblSchemaById(tblId) { const _perfStart = recordPerfStart(); - const ncTbl = await api.dbTable.read(tblId); + // const ncTbl = await api.dbTable.read(tblId); + const ncTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: tblId, + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.read'); updateNcTblSchema(ncTbl); @@ -2060,8 +2266,12 @@ export default async ( // insert filters for (let i = 0; i < ncFilters.length; i++) { const _perfStart = recordPerfStart(); - await api.dbTableFilter.create(viewId, { - ...ncFilters[i], + // await api.dbTableFilter.create(viewId, { + // ...ncFilters[i], + // }); + await filterService.filterCreate({ + viewId: viewId, + filter: ncFilters[i], }); recordPerfStats(_perfStart, 'dbTableFilter.create'); @@ -2076,9 +2286,16 @@ export default async ( if (columnId) { const _perfStart = recordPerfStart(); - await api.dbTableSort.create(viewId, { - fk_column_id: columnId, - direction: s.sortSet[i].ascending ? 'asc' : 'desc', + // await api.dbTableSort.create(viewId, { + // fk_column_id: columnId, + // direction: s.sortSet[i].ascending ? 'asc' : 'desc', + // }); + await sortService.sortCreate({ + viewId: viewId, + sort: { + fk_column_id: columnId, + direction: s.sortSet[i].ascending ? 'asc' : 'desc', + }, }); recordPerfStats(_perfStart, 'dbTableSort.create'); } @@ -2100,13 +2317,21 @@ export default async ( const _perfStart = recordPerfStart(); if (viewType === 'form') { - viewDetails = (await api.dbView.formRead(viewId)).columns; + // viewDetails = (await api.dbView.formRead(viewId)).columns; + viewDetails = (await formViewService.formViewGet({ formViewId: viewId })) + .columns; recordPerfStats(_perfStart, 'dbView.formRead'); } else if (viewType === 'gallery') { - viewDetails = (await api.dbView.galleryRead(viewId)).columns; + // viewDetails = (await api.dbView.galleryRead(viewId)).columns; + viewDetails = ( + await galleryViewService.galleryViewGet({ + galleryViewId: viewId, + }) + ).columns; recordPerfStats(_perfStart, 'dbView.galleryRead'); } else { - viewDetails = await api.dbView.gridColumnsList(viewId); + // viewDetails = await api.dbView.gridColumnsList(viewId); + viewDetails = await viewColumnService.columnList({ viewId: viewId }); recordPerfStats(_perfStart, 'dbView.gridColumnsList'); } @@ -2127,9 +2352,17 @@ export default async ( // first two positions held by record id & record hash const _perfStart = recordPerfStart(); - await api.dbViewColumn.update(viewId, ncViewColumnId, { - show: false, - order: j + 1 + c.length, + // await api.dbViewColumn.update(viewId, ncViewColumnId, { + // show: false, + // order: j + 1 + c.length, + // }); + await viewColumnService.columnUpdate({ + viewId: viewId, + columnId: ncViewColumnId, + column: { + show: false, + order: j + 1 + c.length, + }, }); recordPerfStats(_perfStart, 'dbViewColumn.update'); } @@ -2154,12 +2387,21 @@ export default async ( if (x?.required) formData[`required`] = x.required; if (x?.description) formData[`description`] = x.description; const _perfStart = recordPerfStart(); - await api.dbView.formColumnUpdate(ncViewColumnId, formData); + // await api.dbView.formColumnUpdate(ncViewColumnId, formData); + await formViewColumnService.columnUpdate({ + formViewColumnId: ncViewColumnId, + formViewColumn: formData, + }); recordPerfStats(_perfStart, 'dbView.formColumnUpdate'); } } const _perfStart = recordPerfStart(); - await api.dbViewColumn.update(viewId, ncViewColumnId, configData); + // await api.dbViewColumn.update(viewId, ncViewColumnId, configData); + await viewColumnService.columnUpdate({ + viewId: viewId, + columnId: ncViewColumnId, + column: configData, + }); recordPerfStats(_perfStart, 'dbViewColumn.update'); } } @@ -2168,13 +2410,6 @@ export default async ( let recordCnt = 0; try { logBasic('SDK initialized'); - api = new Api({ - baseURL: syncDB.baseURL, - headers: { - 'xc-auth': syncDB.authToken, - }, - }); - logDetailed('Project initialization started'); // delete project if already exists if (debugMode) await init(syncDB); @@ -2254,10 +2489,16 @@ export default async ( try { // await nc_DumpTableSchema(); const _perfStart = recordPerfStart(); - const ncTblList = await api.base.tableList( - ncCreatedProjectSchema.id, - syncDB.baseId - ); + // const ncTblList = await api.base.tableList( + // ncCreatedProjectSchema.id, + // syncDB.baseId + // ); + const ncTblList = { list: [] }; + ncTblList['list'] = await tableService.getAccessibleTables({ + projectId: ncCreatedProjectSchema.id, + baseId: syncDB.baseId, + roles: userRole, + }); recordPerfStats(_perfStart, 'base.tableList'); logBasic('Reading Records...'); @@ -2273,7 +2514,11 @@ export default async ( continue; const _perfStart = recordPerfStart(); - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + // const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + const ncTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: ncTblList.list[i].id, + user: syncDB.user, + }); recordPerfStats(_perfStart, 'dbTable.read'); recordCnt = 0; @@ -2283,7 +2528,6 @@ export default async ( projectName: syncDB.projectName, table: ncTbl, base, - api, logBasic, nocoBaseDataProcessing_v2, sDB: syncDB, @@ -2303,12 +2547,15 @@ export default async ( ) continue; - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + // const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + const ncTbl: any = await tableService.getTableWithAccessibleViews({ + tableId: ncTblList.list[i].id, + user: syncDB.user, + }); rtc.data.nestedLinks += await importLTARData({ table: ncTbl, projectName: syncDB.projectName, - api, base, fields: null, //Object.values(tblLinkGroup).flat(), logBasic, @@ -2317,6 +2564,7 @@ export default async ( records: recordsMap[ncTbl.id], atNcAliasRef, ncLinkMappingTable, + syncDB, }); } @@ -2417,6 +2665,7 @@ export interface AirtableSyncConfig { baseId?: string; apiKey: string; shareId: string; + user: UserType; options: { syncViews: boolean; syncData: boolean; diff --git a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts index faf06482a0..7dcd31d813 100644 --- a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts @@ -1,7 +1,8 @@ import { RelationTypes, UITypes } from 'nocodb-sdk'; +import { bulkDataService, tableService } from '../..'; import EntityMap from './EntityMap'; import type { AirtableBase } from 'airtable/lib/airtable_base'; -import type { Api, TableType } from 'nocodb-sdk'; +import type { TableType } from 'nocodb-sdk'; const BULK_DATA_BATCH_SIZE = 500; const ASSOC_BULK_DATA_BATCH_SIZE = 1000; @@ -70,7 +71,6 @@ export async function importData({ projectName, table, base, - api, nocoBaseDataProcessing_v2, sDB, logDetailed = (_str) => {}, @@ -82,7 +82,6 @@ export async function importData({ base: AirtableBase; logBasic: (string) => void; logDetailed: (string) => void; - api: Api; nocoBaseDataProcessing_v2; sDB; }): Promise { @@ -116,12 +115,20 @@ export async function importData({ if (tempData.length >= BULK_DATA_BATCH_SIZE) { let insertArray = tempData.splice(0, tempData.length); - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - table.id, - insertArray - ); + // await api.dbTableRow.bulkCreate( + // 'nc', + // projectName, + // table.id, + // insertArray + // ); + + await bulkDataService.bulkDataInsert({ + projectName: projectName, + tableName: table.title, + body: insertArray, + cookie: {}, + }); + logBasic( `:: Importing '${ table.title @@ -142,12 +149,20 @@ export async function importData({ readable.on('end', async () => { await Promise.all(promises); if (tempData.length > 0) { - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - table.id, - tempData - ); + // await api.dbTableRow.bulkCreate( + // 'nc', + // projectName, + // table.id, + // tempData + // ); + + await bulkDataService.bulkDataInsert({ + projectName: projectName, + tableName: table.title, + body: tempData, + cookie: {}, + }); + logBasic( `:: Importing '${ table.title @@ -174,7 +189,6 @@ export async function importLTARData({ table, fields, base, - api, projectName, insertedAssocRef = {}, logDetailed = (_str) => {}, @@ -182,6 +196,7 @@ export async function importLTARData({ records, atNcAliasRef, ncLinkMappingTable, + syncDB, }: { projectName: string; table: { title?: string; id?: string }; @@ -189,7 +204,6 @@ export async function importLTARData({ base: AirtableBase; logDetailed: (string) => void; logBasic: (string) => void; - api: Api; insertedAssocRef: { [assocTableId: string]: boolean }; records?: EntityMap; atNcAliasRef: { @@ -198,6 +212,7 @@ export async function importLTARData({ }; }; ncLinkMappingTable: Record>[]; + syncDB; }) { const assocTableMetas: Array<{ modelMeta: { id?: string; title?: string }; @@ -215,7 +230,11 @@ export async function importLTARData({ logBasic, })); - const modelMeta: any = await api.dbTable.read(table.id); + // const modelMeta: any = await api.dbTable.read(table.id); + const modelMeta: any = await tableService.getTableWithAccessibleViews({ + tableId: table.id, + user: syncDB.user, + }); for (const colMeta of modelMeta.columns) { // skip columns which are not LTAR and Many to many @@ -235,9 +254,15 @@ export async function importLTARData({ // mark as inserted insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; - const assocModelMeta: TableType = (await api.dbTable.read( - colMeta.colOptions.fk_mm_model_id - )) as any; + // const assocModelMeta: TableType = (await api.dbTable.read( + // colMeta.colOptions.fk_mm_model_id + // )) as any; + + const assocModelMeta: TableType = + (await tableService.getTableWithAccessibleViews({ + tableId: colMeta.colOptions.fk_mm_model_id, + user: syncDB.user, + })) as any; // extract associative table and columns meta assocTableMetas.push({ @@ -291,12 +316,19 @@ export async function importLTARData({ )}` ); - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - assocMeta.modelMeta.id, - insertArray - ); + // await api.dbTableRow.bulkCreate( + // 'nc', + // projectName, + // assocMeta.modelMeta.id, + // insertArray + // ); + + await bulkDataService.bulkDataInsert({ + projectName: projectName, + tableName: table.title, + body: insertArray, + cookie: {}, + }); importedCount += insertArray.length; insertArray = []; @@ -319,12 +351,19 @@ export async function importLTARData({ )}` ); - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - assocMeta.modelMeta.id, - assocTableData - ); + // await api.dbTableRow.bulkCreate( + // 'nc', + // projectName, + // assocMeta.modelMeta.id, + // assocTableData + // ); + + await bulkDataService.bulkDataInsert({ + projectName: projectName, + tableName: table.title, + body: assocTableData, + cookie: {}, + }); importedCount += assocTableData.length; assocTableData = []; From 80b122afa9d41c9f1bf8e64e10c8474ecd69b236 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Sat, 11 Mar 2023 22:36:08 +0530 Subject: [PATCH 2/7] refactor: comments cleanup Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../src/lib/services/sync/helpers/job.ts | 236 +----------------- .../sync/helpers/readAndProcessData.ts | 32 --- 2 files changed, 3 insertions(+), 265 deletions(-) diff --git a/packages/nocodb/src/lib/services/sync/helpers/job.ts b/packages/nocodb/src/lib/services/sync/helpers/job.ts index f95b19132e..27dc7cdfe6 100644 --- a/packages/nocodb/src/lib/services/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/services/sync/helpers/job.ts @@ -1,6 +1,5 @@ import { promisify } from 'util'; import { UITypes } from 'nocodb-sdk'; -// import * as sMap from './syncMap'; import Airtable from 'airtable'; import jsonfile from 'jsonfile'; @@ -307,16 +306,6 @@ export default async ( }; } - // aTbl: retrieve table name from table ID - // - // @ts-ignore - function aTbl_getTableName(tblId) { - const sheetObj = g_aTblSchema.find((tbl) => tbl.id === tblId); - return { - tn: sheetObj.name, - }; - } - const ncSchema = { tables: [], tablesById: {}, @@ -368,13 +357,6 @@ export default async ( // retrieve nc column schema from using aTbl field ID as reference // async function nc_getColumnSchema(aTblFieldId) { - // let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); - // let aTblField = aTbl_getColumnName(aTblFieldId); - // let ncTblId = ncTblList.list.filter(x => x.title === aTblField.tn)[0].id; - // let ncTbl = await api.dbTable.read(ncTblId); - // let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn); - // return ncCol; - const ncTblId = await sMap.getNcParentFromAtId(aTblFieldId); const ncColId = await sMap.getNcIdFromAtId(aTblFieldId); @@ -388,11 +370,6 @@ export default async ( // optimize: create a look-up table & re-use information // async function nc_getTableSchema(tableName) { - // let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); - // let ncTblId = ncTblList.list.filter(x => x.title === tableName)[0].id; - // let ncTbl = await api.dbTable.read(ncTblId); - // return ncTbl; - return ncSchema.tables.find((x) => x.title === tableName); } @@ -404,7 +381,6 @@ export default async ( projectId?: string; }) { // delete 'sample' project if already exists - // const x = await api.project.list(); const x = { list: [] }; x['list'] = await projectService.projectList({ user: { id: syncDB.user.id, roles: syncDB.user.roles }, @@ -412,7 +388,6 @@ export default async ( const sampleProj = x.list.find((a) => a.title === projectName); if (sampleProj) { - // await api.project.delete(sampleProj.id); await projectService.projectSoftDelete({ projectId: sampleProj.id, }); @@ -461,11 +436,6 @@ export default async ( case 'date': if (col.typeOptions?.isDateTime) ncType = UITypes.DateTime; break; - - // case 'barcode': - // case 'button': - // ncType = UITypes.SingleLineText; - // break; } return ncType; @@ -492,7 +462,9 @@ export default async ( (value as any).name = 'nc_empty'; } // enumerate duplicates (we don't allow them) - // TODO fix record mapping (this causes every record to map first option, we can't handle them using data api as they don't provide option id within data we might instead get the correct mapping from schema file ) + // TODO fix record mapping (this causes every record to map first option, + // we can't handle them using data api as they don't provide option id + // within data we might instead get the correct mapping from schema file ) let dupNo = 1; const defaultName = (value as any).name; while ( @@ -602,13 +574,6 @@ export default async ( continue; } - // populate cdf (column default value) if configured - // if (col?.default) { - // if (typeof col.default === 'string') - // ncCol.cdf = `'${col.default.replace?.(/'/g, "\\'")}'`; - // else ncCol.cdf = col.default; - // } - // change from default 'tinytext' as airtable allows more than 255 characters // for single line text column type if (col.type === 'text') ncCol.dt = 'text'; @@ -665,11 +630,6 @@ export default async ( logDetailed(`NC API: base.tableCreate ${tables[idx].title}`); let _perfStart = recordPerfStart(); - // const table: any = await api.base.tableCreate( - // ncCreatedProjectSchema.id, - // syncDB.baseId, - // tables[idx] - // ); const table = await tableService.tableCreate({ baseId: syncDB.baseId, projectId: ncCreatedProjectSchema.id, @@ -699,7 +659,6 @@ export default async ( // update default view name- to match it to airtable view name logDetailed(`NC API: dbView.list ${table.id}`); _perfStart = recordPerfStart(); - // const view = await api.dbView.list(table.id); const view = { list: [] }; view['list'] = await viewService.viewList({ tableId: table.id, @@ -710,9 +669,6 @@ export default async ( const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid'); logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); _perfStart = recordPerfStart(); - // await api.dbView.update(view.list[0].id, { - // title: aTbl_grid.name, - // }); await viewService.viewUpdate({ viewId: view.list[0].id, view: { title: aTbl_grid.name }, @@ -763,7 +719,6 @@ export default async ( // check if link already established? if (!nc_isLinkExists(aTblLinkColumns[i].id)) { // parent table ID - // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); // find child table name from symmetric column ID specified @@ -780,7 +735,6 @@ export default async ( // check if already a column exists with this name? let _perfStart = recordPerfStart(); - // const srcTbl: any = await api.dbTable.read(srcTableId); const srcTbl: any = await tableService.getTableWithAccessibleViews({ tableId: srcTableId, user: syncDB.user, @@ -801,17 +755,6 @@ export default async ( `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` ); _perfStart = recordPerfStart(); - // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - // uidt: UITypes.LinkToAnotherRecord, - // title: ncName.title, - // column_name: ncName.column_name, - // parentId: srcTableId, - // childId: childTableId, - // type: 'mm', - // // aTblLinkColumns[i].typeOptions.relationship === 'many' - // // ? 'mm' - // // : 'hm' - // }); const ncTbl: any = await columnService.columnAdd({ tableId: srcTableId, column: { @@ -821,9 +764,6 @@ export default async ( parentId: srcTableId, childId: childTableId, type: 'mm', - // aTblLinkColumns[i].typeOptions.relationship === 'many' - // ? 'mm' - // : 'hm' }, req: { user: syncDB.user.email, @@ -876,9 +816,6 @@ export default async ( ); let _perfStart = recordPerfStart(); - // const childTblSchema: any = await api.dbTable.read( - // ncLinkMappingTable[x].nc.childId - // ); const childTblSchema: any = await tableService.getTableWithAccessibleViews({ tableId: ncLinkMappingTable[x].nc.childId, @@ -887,9 +824,6 @@ export default async ( recordPerfStats(_perfStart, 'dbTable.read'); _perfStart = recordPerfStart(); - // const parentTblSchema: any = await api.dbTable.read( - // ncLinkMappingTable[x].nc.parentId - // ); const parentTblSchema: any = await tableService.getTableWithAccessibleViews({ tableId: ncLinkMappingTable[x].nc.parentId, @@ -897,10 +831,6 @@ export default async ( }); recordPerfStats(_perfStart, 'dbTable.read'); - // fix me - // let childTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.childId] - // let parentTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.parentId] - let parentLinkColumn = parentTblSchema.columns.find( (col) => col.title === ncLinkMappingTable[x].nc.title ); @@ -973,14 +903,6 @@ export default async ( `NC API: dbTableColumn.update rename symmetric column ${ncName.title}` ); _perfStart = recordPerfStart(); - // const ncTbl: any = await api.dbTableColumn.update( - // childLinkColumn.id, - // { - // ...childLinkColumn, - // title: ncName.title, - // column_name: ncName.column_name, - // } - // ); const ncTbl: any = await columnService.columnUpdate({ columnId: childLinkColumn.id, column: { @@ -1002,8 +924,6 @@ export default async ( aTblLinkColumns[i].name + suffix, ncTbl.id ); - - // console.log(res.columns.find(x => x.title === aTblLinkColumns[i].name)) } } } @@ -1018,7 +938,6 @@ export default async ( ); // parent table ID - // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); const srcTableSchema = ncSchema.tablesById[srcTableId]; @@ -1070,13 +989,6 @@ export default async ( logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const _perfStart = recordPerfStart(); - // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - // uidt: UITypes.Lookup, - // title: ncName.title, - // column_name: ncName.column_name, - // fk_relation_column_id: ncRelationColumnId, - // fk_lookup_column_id: ncLookupColumnId, - // }); const ncTbl: any = await columnService.columnAdd({ tableId: srcTableId, column: { @@ -1164,13 +1076,6 @@ export default async ( logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const _perfStart = recordPerfStart(); - // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - // uidt: UITypes.Lookup, - // title: ncName.title, - // column_name: ncName.column_name, - // fk_relation_column_id: ncRelationColumnId, - // fk_lookup_column_id: ncLookupColumnId, - // }); const ncTbl: any = await columnService.columnAdd({ tableId: srcTableId, column: { @@ -1236,7 +1141,6 @@ export default async ( ); // parent table ID - // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); const srcTableSchema = ncSchema.tablesById[srcTableId]; @@ -1254,7 +1158,6 @@ export default async ( const ncRollupFn = getRollupNcFunction( aTblColumns[i].typeOptions.formulaTextParsed ); - // const ncRollupFn = ''; if (ncRollupFn === '' || ncRollupFn === undefined) { updateMigrationSkipLog( @@ -1321,14 +1224,6 @@ export default async ( logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); const _perfStart = recordPerfStart(); - // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - // uidt: UITypes.Rollup, - // title: ncName.title, - // column_name: ncName.column_name, - // fk_relation_column_id: ncRelationColumnId, - // fk_rollup_column_id: ncRollupColumnId, - // rollup_function: ncRollupFn, - // }); const ncTbl: any = await columnService.columnAdd({ tableId: srcTableId, column: { @@ -1394,13 +1289,6 @@ export default async ( logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const _perfStart = recordPerfStart(); - // const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - // uidt: UITypes.Lookup, - // title: ncName.title, - // column_name: ncName.column_name, - // fk_relation_column_id: ncRelationColumnId, - // fk_lookup_column_id: ncLookupColumnId, - // }); const ncTbl: any = await columnService.columnAdd({ tableId: srcTableId, column: { @@ -1449,7 +1337,6 @@ export default async ( if (ncColId) { logDetailed(`NC API: dbTableColumn.primaryColumnSet`); const _perfStart = recordPerfStart(); - // await api.dbTableColumn.primaryColumnSet(ncColId); await columnService.columnSetAsPrimary({ columnId: ncColId }); recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet'); @@ -1467,18 +1354,15 @@ export default async ( const _perfStart = recordPerfStart(); if (viewType === 'form') { - // viewDetails = (await api.dbView.formRead(viewId)).columns; viewDetails = (await formViewService.formViewGet({ formViewId: viewId })) .columns; recordPerfStats(_perfStart, 'dbView.formRead'); } else if (viewType === 'gallery') { - // viewDetails = (await api.dbView.galleryRead(viewId)).columns; viewDetails = ( await galleryViewService.galleryViewGet({ galleryViewId: viewId }) ).columns; recordPerfStats(_perfStart, 'dbView.galleryRead'); } else { - // viewDetails = await api.dbView.gridColumnsList(viewId); viewDetails = await viewColumnService.columnList({ viewId: viewId }); recordPerfStats(_perfStart, 'dbView.gridColumnsList'); } @@ -1607,17 +1491,6 @@ export default async ( ?.map((a) => a.filename?.split('?')?.[0]) .join(', ')}` ); - // tempArr = await api.storage.uploadByUrl( - // { - // path: `noco/${sDB.projectName}/${table.title}/${key}`, - // }, - // value?.map((attachment) => ({ - // fileName: attachment.filename?.split('?')?.[0], - // url: attachment.url, - // size: attachment.size, - // mimetype: attachment.type, - // })) - // ); tempArr = await attachmentService.uploadViaURL({ path: `noco/${sDB.projectName}/${table.title}/${key}`, urls: value?.map((attachment) => ({ @@ -1665,8 +1538,6 @@ export default async ( }) .eachPage( async function page(records, fetchNextPage) { - // console.log(JSON.stringify(records, null, 2)); - // This function (`page`) will get called for each page of records. // records.forEach(record => callback(table, record)); logBasic( @@ -1706,9 +1577,6 @@ export default async ( // create empty project (XC-DB) logDetailed(`Create Project: ${projName}`); const _perfStart = recordPerfStart(); - // ncCreatedProjectSchema = await api.project.create({ - // title: projName, - // }); ncCreatedProjectSchema = await projectService.projectCreate({ project: { title: projName }, @@ -1722,7 +1590,6 @@ export default async ( // create empty project (XC-DB) logDetailed(`Getting project meta: ${projId}`); const _perfStart = recordPerfStart(); - // ncCreatedProjectSchema = await api.project.read(projId); ncCreatedProjectSchema = await projectService.getProjectWithInfo({ projectId: projId, }); @@ -1757,7 +1624,6 @@ export default async ( logDetailed(`NC API dbView.galleryCreate :: ${viewName}`); const _perfStart = recordPerfStart(); - // await api.dbView.galleryCreate(tblId, { title: viewName }); await galleryViewService.galleryViewCreate({ tableId: tblId, gallery: { @@ -1767,9 +1633,6 @@ export default async ( recordPerfStats(_perfStart, 'dbView.galleryCreate'); await updateNcTblSchemaById(tblId); - // syncLog(`[${idx+1}/${aTblSchema.length}][Gallery View][${i+1}/${galleryViews.length}] Create ${viewName}`) - - // await nc_configureFields(g.id, vData, aTblSchema[idx].name, viewName, 'gallery'); } } } @@ -1891,9 +1754,6 @@ export default async ( if (i > 0) { logDetailed(`NC API dbView.gridCreate :: ${viewName}`); const _perfStart = recordPerfStart(); - // const viewCreated = await api.dbView.gridCreate(tblId, { - // title: viewName, - // }); const viewCreated = await gridViewService.gridViewCreate({ tableId: tblId, grid: { @@ -1909,11 +1769,9 @@ export default async ( viewName, tblId ); - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Create ${viewName}`) ncViewId = viewCreated.id; } - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Hide columns ${viewName}`) logDetailed(` Configure show/hide columns`); await nc_configureFields( ncViewId, @@ -1925,7 +1783,6 @@ export default async ( // configure filters if (vData?.filters) { - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure filters ${viewName}`) logDetailed(` Configure filter set`); // skip filters if nested @@ -1936,7 +1793,6 @@ export default async ( // configure sort if (vData?.lastSortsApplied?.sortSet.length) { - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure sort ${viewName}`) logDetailed(` Configure sort set`); await nc_configureSort(ncViewId, vData.lastSortsApplied); } @@ -1966,11 +1822,6 @@ export default async ( ); const _perfStart = recordPerfStart(); insertJobs.push( - // api.auth - // .projectUserAdd(ncCreatedProjectSchema.id, { - // email: value.email, - // roles: userRoles[value.permissionLevel], - // }) projectUserService .userInvite({ projectId: ncCreatedProjectSchema.id, @@ -2005,7 +1856,6 @@ export default async ( async function updateNcTblSchemaById(tblId) { const _perfStart = recordPerfStart(); - // const ncTbl = await api.dbTable.read(tblId); const ncTbl: any = await tableService.getTableWithAccessibleViews({ tableId: tblId, user: syncDB.user, @@ -2266,9 +2116,6 @@ export default async ( // insert filters for (let i = 0; i < ncFilters.length; i++) { const _perfStart = recordPerfStart(); - // await api.dbTableFilter.create(viewId, { - // ...ncFilters[i], - // }); await filterService.filterCreate({ viewId: viewId, filter: ncFilters[i], @@ -2286,10 +2133,6 @@ export default async ( if (columnId) { const _perfStart = recordPerfStart(); - // await api.dbTableSort.create(viewId, { - // fk_column_id: columnId, - // direction: s.sortSet[i].ascending ? 'asc' : 'desc', - // }); await sortService.sortCreate({ viewId: viewId, sort: { @@ -2317,12 +2160,10 @@ export default async ( const _perfStart = recordPerfStart(); if (viewType === 'form') { - // viewDetails = (await api.dbView.formRead(viewId)).columns; viewDetails = (await formViewService.formViewGet({ formViewId: viewId })) .columns; recordPerfStats(_perfStart, 'dbView.formRead'); } else if (viewType === 'gallery') { - // viewDetails = (await api.dbView.galleryRead(viewId)).columns; viewDetails = ( await galleryViewService.galleryViewGet({ galleryViewId: viewId, @@ -2330,7 +2171,6 @@ export default async ( ).columns; recordPerfStats(_perfStart, 'dbView.galleryRead'); } else { - // viewDetails = await api.dbView.gridColumnsList(viewId); viewDetails = await viewColumnService.columnList({ viewId: viewId }); recordPerfStats(_perfStart, 'dbView.gridColumnsList'); } @@ -2343,19 +2183,10 @@ export default async ( const ncViewColumnId = viewDetails.find( (x) => x.fk_column_id === ncColumnId )?.id; - // const ncViewColumnId = await nc_getViewColumnId( - // viewId, - // viewType, - // ncColumnId - // ); if (ncViewColumnId === undefined) continue; // first two positions held by record id & record hash const _perfStart = recordPerfStart(); - // await api.dbViewColumn.update(viewId, ncViewColumnId, { - // show: false, - // order: j + 1 + c.length, - // }); await viewColumnService.columnUpdate({ viewId: viewId, columnId: ncViewColumnId, @@ -2387,7 +2218,6 @@ export default async ( if (x?.required) formData[`required`] = x.required; if (x?.description) formData[`description`] = x.description; const _perfStart = recordPerfStart(); - // await api.dbView.formColumnUpdate(ncViewColumnId, formData); await formViewColumnService.columnUpdate({ formViewColumnId: ncViewColumnId, formViewColumn: formData, @@ -2396,7 +2226,6 @@ export default async ( } } const _perfStart = recordPerfStart(); - // await api.dbViewColumn.update(viewId, ncViewColumnId, configData); await viewColumnService.columnUpdate({ viewId: viewId, columnId: ncViewColumnId, @@ -2489,10 +2318,6 @@ export default async ( try { // await nc_DumpTableSchema(); const _perfStart = recordPerfStart(); - // const ncTblList = await api.base.tableList( - // ncCreatedProjectSchema.id, - // syncDB.baseId - // ); const ncTblList = { list: [] }; ncTblList['list'] = await tableService.getAccessibleTables({ projectId: ncCreatedProjectSchema.id, @@ -2514,7 +2339,6 @@ export default async ( continue; const _perfStart = recordPerfStart(); - // const ncTbl = await api.dbTable.read(ncTblList.list[i].id); const ncTbl: any = await tableService.getTableWithAccessibleViews({ tableId: ncTblList.list[i].id, user: syncDB.user, @@ -2522,7 +2346,6 @@ export default async ( recordPerfStats(_perfStart, 'dbTable.read'); recordCnt = 0; - // await nocoReadData(syncDB, ncTbl); recordsMap[ncTbl.id] = await importData({ projectName: syncDB.projectName, @@ -2567,59 +2390,6 @@ export default async ( syncDB, }); } - - if (storeLinks) { - // const insertJobs: Promise[] = []; - // for (const [pTitle, v] of Object.entries(ncLinkDataStore)) { - // logBasic(`:: ${pTitle}`); - // for (const [, record] of Object.entries(v)) { - // const tbl = ncTblList.list.find(a => a.title === pTitle); - // await nocoLinkProcessing(syncDB.projectName, tbl, record, 0); - // // insertJobs.push( - // // nocoLinkProcessing(syncDB.projectName, tbl, record, 0) - // // ); - // } - // } - // await Promise.all(insertJobs); - // await nocoLinkProcessing(syncDB.projectName, 0, 0, 0); - } else { - // // create link groups (table: link fields) - // // const tblLinkGroup = {}; - // // for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { - // // const x = ncLinkMappingTable[idx]; - // // if (tblLinkGroup[x.aTbl.tblId] === undefined) - // // tblLinkGroup[x.aTbl.tblId] = [x.aTbl.name]; - // // else tblLinkGroup[x.aTbl.tblId].push(x.aTbl.name); - // // } - // // - // // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); - // // - // // await importLTARData({ - // // table: ncTbl, - // // projectName: syncDB.projectName, - // // api, - // // base, - // // fields: Object.values(tblLinkGroup).flat(), - // // logBasic - // // }); - // for (const [k, v] of Object.entries(tblLinkGroup)) { - // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); - // - // // not a migrated table, skip - // if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) - // continue; - // - // recordCnt = 0; - // await nocoReadDataSelected( - // syncDB.projectName, - // ncTbl, - // async (projName, table, record, _field) => { - // await nocoLinkProcessing(projName, table, record, _field); - // }, - // v - // ); - // } - } } catch (error) { logDetailed( `There was an error while migrating data! Please make sure your API key (${syncDB.apiKey}) is correct.` diff --git a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts index 7dcd31d813..f45ec91c33 100644 --- a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts @@ -115,12 +115,6 @@ export async function importData({ if (tempData.length >= BULK_DATA_BATCH_SIZE) { let insertArray = tempData.splice(0, tempData.length); - // await api.dbTableRow.bulkCreate( - // 'nc', - // projectName, - // table.id, - // insertArray - // ); await bulkDataService.bulkDataInsert({ projectName: projectName, @@ -149,13 +143,6 @@ export async function importData({ readable.on('end', async () => { await Promise.all(promises); if (tempData.length > 0) { - // await api.dbTableRow.bulkCreate( - // 'nc', - // projectName, - // table.id, - // tempData - // ); - await bulkDataService.bulkDataInsert({ projectName: projectName, tableName: table.title, @@ -230,7 +217,6 @@ export async function importLTARData({ logBasic, })); - // const modelMeta: any = await api.dbTable.read(table.id); const modelMeta: any = await tableService.getTableWithAccessibleViews({ tableId: table.id, user: syncDB.user, @@ -254,10 +240,6 @@ export async function importLTARData({ // mark as inserted insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; - // const assocModelMeta: TableType = (await api.dbTable.read( - // colMeta.colOptions.fk_mm_model_id - // )) as any; - const assocModelMeta: TableType = (await tableService.getTableWithAccessibleViews({ tableId: colMeta.colOptions.fk_mm_model_id, @@ -316,13 +298,6 @@ export async function importLTARData({ )}` ); - // await api.dbTableRow.bulkCreate( - // 'nc', - // projectName, - // assocMeta.modelMeta.id, - // insertArray - // ); - await bulkDataService.bulkDataInsert({ projectName: projectName, tableName: table.title, @@ -351,13 +326,6 @@ export async function importLTARData({ )}` ); - // await api.dbTableRow.bulkCreate( - // 'nc', - // projectName, - // assocMeta.modelMeta.id, - // assocTableData - // ); - await bulkDataService.bulkDataInsert({ projectName: projectName, tableName: table.title, From fdaf5ee272f7a9f812d5d401f234f61a2fdeae0d Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Mon, 13 Mar 2023 11:07:34 +0530 Subject: [PATCH 3/7] test: airtable suite corrections Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../src/lib/services/sync/helpers/readAndProcessData.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts index f45ec91c33..37f91605fb 100644 --- a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts @@ -298,12 +298,13 @@ export async function importLTARData({ )}` ); - await bulkDataService.bulkDataInsert({ + const x = await bulkDataService.bulkDataInsert({ projectName: projectName, - tableName: table.title, + tableName: assocMeta.modelMeta.title, body: insertArray, cookie: {}, }); + console.log(x); importedCount += insertArray.length; insertArray = []; @@ -328,7 +329,7 @@ export async function importLTARData({ await bulkDataService.bulkDataInsert({ projectName: projectName, - tableName: table.title, + tableName: assocMeta.modelMeta.title, body: assocTableData, cookie: {}, }); From f7bd2caaa4083e504db7242e08690cc9b018abb4 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Mon, 13 Mar 2023 14:06:20 +0530 Subject: [PATCH 4/7] refactor: remove baseURL env variable Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../content/en/getting-started/environment-variables.md | 1 - packages/nocodb/src/lib/controllers/sync/import.ctl.ts | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/noco-docs/content/en/getting-started/environment-variables.md b/packages/noco-docs/content/en/getting-started/environment-variables.md index 5ce104419c..b15712a99b 100644 --- a/packages/noco-docs/content/en/getting-started/environment-variables.md +++ b/packages/noco-docs/content/en/getting-started/environment-variables.md @@ -43,7 +43,6 @@ For production usecases, it is **recommended** to configure | NC_REDIS_URL | Custom Redis URL. Example: `redis://:authpassword@127.0.0.1:6380/4` | Meta data will be stored in memory | | | NC_DISABLE_ERR_REPORT | Disable error reporting | | | | NC_DISABLE_CACHE | To be used only while debugging. On setting this to `true` - meta data be fetched from db instead of redis/cache. | `false` | | -| NC_BASEURL_INTERNAL | Used as base url for internal(server) API calls | Default value in docker will be `http://localhost:$PORT` and in all other case it's populated from request object | | | AWS_ACCESS_KEY_ID | For Litestream - S3 access key id | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 | | | AWS_SECRET_ACCESS_KEY | For Litestream - S3 secret access key | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 | | | AWS_BUCKET | For Litestream - S3 bucket | If Litestream is configured and `NC_DB` is not present. SQLite gets backed up to S3 | | diff --git a/packages/nocodb/src/lib/controllers/sync/import.ctl.ts b/packages/nocodb/src/lib/controllers/sync/import.ctl.ts index c497d80735..fb3c06ecda 100644 --- a/packages/nocodb/src/lib/controllers/sync/import.ctl.ts +++ b/packages/nocodb/src/lib/controllers/sync/import.ctl.ts @@ -105,9 +105,7 @@ export default ( // if environment value avail use it // or if it's docker construct using `PORT` - if (process.env.NC_BASEURL_INTERNAL) { - baseURL = process.env.NC_BASEURL_INTERNAL; - } else if (process.env.NC_DOCKER) { + if (process.env.NC_DOCKER) { baseURL = `http://localhost:${process.env.PORT || 8080}`; } From 5e210111a6d3585383281ad3239e7981a6d6b299 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Mon, 13 Mar 2023 18:01:30 +0530 Subject: [PATCH 5/7] chore: remove comment Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../nocodb/src/lib/services/sync/helpers/readAndProcessData.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts index 37f91605fb..5a4ba5cdd1 100644 --- a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts @@ -298,13 +298,12 @@ export async function importLTARData({ )}` ); - const x = await bulkDataService.bulkDataInsert({ + await bulkDataService.bulkDataInsert({ projectName: projectName, tableName: assocMeta.modelMeta.title, body: insertArray, cookie: {}, }); - console.log(x); importedCount += insertArray.length; insertArray = []; From d1fd28c9bb807334c6b9f587835716e0bba35a10 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Tue, 14 Mar 2023 16:52:56 +0530 Subject: [PATCH 6/7] refactor: use of shorthand in code Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../nocodb/src/lib/services/sync/helpers/readAndProcessData.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts index 5a4ba5cdd1..66e30d49c9 100644 --- a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts @@ -299,7 +299,7 @@ export async function importLTARData({ ); await bulkDataService.bulkDataInsert({ - projectName: projectName, + projectName, tableName: assocMeta.modelMeta.title, body: insertArray, cookie: {}, From 2b8dd8c4f2728770108dc7124282f22bae5fc7d0 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:01:07 +0530 Subject: [PATCH 7/7] refactor: use of shorthand in code Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../src/lib/services/sync/helpers/readAndProcessData.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts index 66e30d49c9..e9c35fad68 100644 --- a/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts @@ -117,7 +117,7 @@ export async function importData({ let insertArray = tempData.splice(0, tempData.length); await bulkDataService.bulkDataInsert({ - projectName: projectName, + projectName, tableName: table.title, body: insertArray, cookie: {}, @@ -144,7 +144,7 @@ export async function importData({ await Promise.all(promises); if (tempData.length > 0) { await bulkDataService.bulkDataInsert({ - projectName: projectName, + projectName, tableName: table.title, body: tempData, cookie: {}, @@ -327,7 +327,7 @@ export async function importLTARData({ ); await bulkDataService.bulkDataInsert({ - projectName: projectName, + projectName, tableName: assocMeta.modelMeta.title, body: assocTableData, cookie: {},