github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

refactor: use service api's in airtable import 

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>
pull/5280/head
Raju Udava 2 years ago
parent
0cfc630b72
commit
636e9babb9
4 changed files with 441 additions and 152 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      packages/nocodb/src/lib/controllers/sync/import.ctl.ts
  2. 4
      packages/nocodb/src/lib/services/column.svc.ts
  3. 483
      packages/nocodb/src/lib/services/sync/helpers/job.ts
  4. 105
      packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts

1
packages/nocodb/src/lib/controllers/sync/import.ctl.ts
Unescape View File

@ -119,6 +119,7 @@ export default (
baseId: syncSource.base_id, baseId: syncSource.base_id,
authToken: token, authToken: token,
baseURL, baseURL,
user: user,
}); });
}, 1000); }, 1000);

4
packages/nocodb/src/lib/services/column.svc.ts
Unescape View File

@ -836,7 +836,7 @@ export async function columnSetAsPrimary(param: { columnId: string }) {
} }
export async function columnAdd(param: { export async function columnAdd(param: {
req?: any; req: any;
tableId: string; tableId: string;
column: ColumnReqType; column: ColumnReqType;
}) { }) {
@ -1111,7 +1111,7 @@ export async function columnAdd(param: {
project_id: base.project_id, project_id: base.project_id,
op_type: AuditOperationTypes.TABLE_COLUMN, op_type: AuditOperationTypes.TABLE_COLUMN,
op_sub_type: AuditOperationSubTypes.CREATED, op_sub_type: AuditOperationSubTypes.CREATED,
user: param?.req?.user?.email, user: param?.req.user?.email,
description: `created column ${colBody.column_name} with alias ${colBody.title} from table ${table.table_name}`, description: `created column ${colBody.column_name} with alias ${colBody.title} from table ${table.table_name}`,
ip: param?.req.clientIp, ip: param?.req.clientIp,
}).then(() => {}); }).then(() => {});

483
packages/nocodb/src/lib/services/sync/helpers/job.ts
Unescape View File

@ -2,8 +2,6 @@ import { promisify } from 'util';
import { UITypes } from 'nocodb-sdk'; import { UITypes } from 'nocodb-sdk';
// import * as sMap from './syncMap'; // import * as sMap from './syncMap';
import { Api } from 'nocodb-sdk';
import Airtable from 'airtable'; import Airtable from 'airtable';
import jsonfile from 'jsonfile'; import jsonfile from 'jsonfile';
import hash from 'object-hash'; import hash from 'object-hash';
@ -12,10 +10,26 @@ import { T } from 'nc-help';
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc'; import utc from 'dayjs/plugin/utc';
import tinycolor from 'tinycolor2'; import tinycolor from 'tinycolor2';
import {
attachmentService,
columnService,
filterService,
formViewColumnService,
formViewService,
galleryViewService,
gridViewService,
projectService,
projectUserService,
sortService,
tableService,
viewColumnService,
viewService,
} from '../..';
import FetchAT from './fetchAT'; import FetchAT from './fetchAT';
import { importData, importLTARData } from './readAndProcessData'; import { importData, importLTARData } from './readAndProcessData';
import EntityMap from './EntityMap'; import EntityMap from './EntityMap';
import type { UserType } from 'nocodb-sdk';
const writeJsonFileAsync = promisify(jsonfile.writeFile); const writeJsonFileAsync = promisify(jsonfile.writeFile);
@ -74,6 +88,9 @@ export default async (
) => { ) => {
const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent'); const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent');
await sMapEM.init(); await sMapEM.init();
const userRole = syncDB.user.roles
.split(',')
.reduce((rolesObj, role) => ({ [role]: true, ...rolesObj }), {});
const sMap = { const sMap = {
// static mapping records between aTblId && ncId // static mapping records between aTblId && ncId
@ -121,7 +138,6 @@ export default async (
const enableErrorLogs = false; const enableErrorLogs = false;
const generate_migrationStats = true; const generate_migrationStats = true;
const debugMode = false; const debugMode = false;
let api: Api<any>;
let g_aTblSchema = []; let g_aTblSchema = [];
let ncCreatedProjectSchema: any = {}; let ncCreatedProjectSchema: any = {};
const ncLinkMappingTable: any[] = []; const ncLinkMappingTable: any[] = [];
@ -325,12 +341,24 @@ export default async (
// @ts-ignore // @ts-ignore
async function nc_DumpTableSchema() { async function nc_DumpTableSchema() {
console.log('['); console.log('[');
const ncTblList = await api.base.tableList( // const ncTblList = await api.base.tableList(
ncCreatedProjectSchema.id, // ncCreatedProjectSchema.id,
syncDB.baseId // syncDB.baseId
); // );
const ncTblList = { list: [] };
ncTblList['list'] = await tableService.getAccessibleTables({
projectId: ncCreatedProjectSchema.id,
baseId: syncDB.baseId,
roles: userRole,
});
for (let i = 0; i < ncTblList.list.length; i++) { for (let i = 0; i < ncTblList.list.length; i++) {
const ncTbl = await api.dbTable.read(ncTblList.list[i].id); // const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
const ncTbl = await tableService.getTableWithAccessibleViews({
tableId: ncTblList.list[i].id,
user: syncDB.user,
});
console.log(JSON.stringify(ncTbl, null, 2)); console.log(JSON.stringify(ncTbl, null, 2));
console.log(','); console.log(',');
} }
@ -376,11 +404,18 @@ export default async (
projectId?: string; projectId?: string;
}) { }) {
// delete 'sample' project if already exists // delete 'sample' project if already exists
const x = await api.project.list(); // const x = await api.project.list();
const x = { list: [] };
x['list'] = await projectService.projectList({
user: { id: syncDB.user.id, roles: syncDB.user.roles },
});
const sampleProj = x.list.find((a) => a.title === projectName); const sampleProj = x.list.find((a) => a.title === projectName);
if (sampleProj) { if (sampleProj) {
await api.project.delete(sampleProj.id); // await api.project.delete(sampleProj.id);
await projectService.projectSoftDelete({
projectId: sampleProj.id,
});
} }
logDetailed('Init'); logDetailed('Init');
} }
@ -630,11 +665,17 @@ export default async (
logDetailed(`NC API: base.tableCreate ${tables[idx].title}`); logDetailed(`NC API: base.tableCreate ${tables[idx].title}`);
let _perfStart = recordPerfStart(); let _perfStart = recordPerfStart();
const table: any = await api.base.tableCreate( // const table: any = await api.base.tableCreate(
ncCreatedProjectSchema.id, // ncCreatedProjectSchema.id,
syncDB.baseId, // syncDB.baseId,
tables[idx] // tables[idx]
); // );
const table = await tableService.tableCreate({
baseId: syncDB.baseId,
projectId: ncCreatedProjectSchema.id,
table: tables[idx],
user: syncDB.user,
});
recordPerfStats(_perfStart, 'dbTable.create'); recordPerfStats(_perfStart, 'dbTable.create');
updateNcTblSchema(table); updateNcTblSchema(table);
@ -658,14 +699,23 @@ export default async (
// update default view name- to match it to airtable view name // update default view name- to match it to airtable view name
logDetailed(`NC API: dbView.list ${table.id}`); logDetailed(`NC API: dbView.list ${table.id}`);
_perfStart = recordPerfStart(); _perfStart = recordPerfStart();
const view = await api.dbView.list(table.id); // const view = await api.dbView.list(table.id);
const view = { list: [] };
view['list'] = await viewService.viewList({
tableId: table.id,
user: { roles: userRole },
});
recordPerfStats(_perfStart, 'dbView.list'); recordPerfStats(_perfStart, 'dbView.list');
const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid'); const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid');
logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`);
_perfStart = recordPerfStart(); _perfStart = recordPerfStart();
await api.dbView.update(view.list[0].id, { // await api.dbView.update(view.list[0].id, {
title: aTbl_grid.name, // title: aTbl_grid.name,
// });
await viewService.viewUpdate({
viewId: view.list[0].id,
view: { title: aTbl_grid.name },
}); });
recordPerfStats(_perfStart, 'dbView.update'); recordPerfStats(_perfStart, 'dbView.update');
@ -730,7 +780,11 @@ export default async (
// check if already a column exists with this name? // check if already a column exists with this name?
let _perfStart = recordPerfStart(); let _perfStart = recordPerfStart();
const srcTbl: any = await api.dbTable.read(srcTableId); // const srcTbl: any = await api.dbTable.read(srcTableId);
const srcTbl: any = await tableService.getTableWithAccessibleViews({
tableId: srcTableId,
user: syncDB.user,
});
recordPerfStats(_perfStart, 'dbTable.read'); recordPerfStats(_perfStart, 'dbTable.read');
// create link // create link
@ -747,16 +801,34 @@ export default async (
`NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}`
); );
_perfStart = recordPerfStart(); _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, { // const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.LinkToAnotherRecord, // uidt: UITypes.LinkToAnotherRecord,
title: ncName.title, // title: ncName.title,
column_name: ncName.column_name, // column_name: ncName.column_name,
parentId: srcTableId, // parentId: srcTableId,
childId: childTableId, // childId: childTableId,
type: 'mm', // type: 'mm',
// aTblLinkColumns[i].typeOptions.relationship === 'many' // // aTblLinkColumns[i].typeOptions.relationship === 'many'
// ? 'mm' // // ? 'mm'
// : 'hm' // // : 'hm'
// });
const ncTbl: any = await columnService.columnAdd({
tableId: srcTableId,
column: {
uidt: UITypes.LinkToAnotherRecord,
title: ncName.title,
column_name: ncName.column_name,
parentId: srcTableId,
childId: childTableId,
type: 'mm',
// aTblLinkColumns[i].typeOptions.relationship === 'many'
// ? 'mm'
// : 'hm'
},
req: {
user: syncDB.user.email,
clientIp: '',
},
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create'); recordPerfStats(_perfStart, 'dbTableColumn.create');
@ -804,15 +876,25 @@ export default async (
); );
let _perfStart = recordPerfStart(); let _perfStart = recordPerfStart();
const childTblSchema: any = await api.dbTable.read( // const childTblSchema: any = await api.dbTable.read(
ncLinkMappingTable[x].nc.childId // ncLinkMappingTable[x].nc.childId
); // );
const childTblSchema: any =
await tableService.getTableWithAccessibleViews({
tableId: ncLinkMappingTable[x].nc.childId,
user: syncDB.user,
});
recordPerfStats(_perfStart, 'dbTable.read'); recordPerfStats(_perfStart, 'dbTable.read');
_perfStart = recordPerfStart(); _perfStart = recordPerfStart();
const parentTblSchema: any = await api.dbTable.read( // const parentTblSchema: any = await api.dbTable.read(
ncLinkMappingTable[x].nc.parentId // ncLinkMappingTable[x].nc.parentId
); // );
const parentTblSchema: any =
await tableService.getTableWithAccessibleViews({
tableId: ncLinkMappingTable[x].nc.parentId,
user: syncDB.user,
});
recordPerfStats(_perfStart, 'dbTable.read'); recordPerfStats(_perfStart, 'dbTable.read');
// fix me // fix me
@ -891,14 +973,22 @@ export default async (
`NC API: dbTableColumn.update rename symmetric column ${ncName.title}` `NC API: dbTableColumn.update rename symmetric column ${ncName.title}`
); );
_perfStart = recordPerfStart(); _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.update( // const ncTbl: any = await api.dbTableColumn.update(
childLinkColumn.id, // childLinkColumn.id,
{ // {
// ...childLinkColumn,
// title: ncName.title,
// column_name: ncName.column_name,
// }
// );
const ncTbl: any = await columnService.columnUpdate({
columnId: childLinkColumn.id,
column: {
...childLinkColumn, ...childLinkColumn,
title: ncName.title, title: ncName.title,
column_name: ncName.column_name, column_name: ncName.column_name,
} },
); });
recordPerfStats(_perfStart, 'dbTableColumn.update'); recordPerfStats(_perfStart, 'dbTableColumn.update');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
@ -980,12 +1070,26 @@ export default async (
logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, { // const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Lookup, // uidt: UITypes.Lookup,
title: ncName.title, // title: ncName.title,
column_name: ncName.column_name, // column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId, // fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId, // fk_lookup_column_id: ncLookupColumnId,
// });
const ncTbl: any = await columnService.columnAdd({
tableId: srcTableId,
column: {
uidt: UITypes.Lookup,
title: ncName.title,
column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId,
},
req: {
user: syncDB.user.email,
clientIp: '',
},
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create'); recordPerfStats(_perfStart, 'dbTableColumn.create');
@ -1060,12 +1164,26 @@ export default async (
logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, { // const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Lookup, // uidt: UITypes.Lookup,
title: ncName.title, // title: ncName.title,
column_name: ncName.column_name, // column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId, // fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId, // fk_lookup_column_id: ncLookupColumnId,
// });
const ncTbl: any = await columnService.columnAdd({
tableId: srcTableId,
column: {
uidt: UITypes.Lookup,
title: ncName.title,
column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId,
},
req: {
user: syncDB.user.email,
clientIp: '',
},
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create'); recordPerfStats(_perfStart, 'dbTableColumn.create');
@ -1203,13 +1321,28 @@ export default async (
logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, { // const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Rollup, // uidt: UITypes.Rollup,
title: ncName.title, // title: ncName.title,
column_name: ncName.column_name, // column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId, // fk_relation_column_id: ncRelationColumnId,
fk_rollup_column_id: ncRollupColumnId, // fk_rollup_column_id: ncRollupColumnId,
rollup_function: ncRollupFn, // rollup_function: ncRollupFn,
// });
const ncTbl: any = await columnService.columnAdd({
tableId: srcTableId,
column: {
uidt: UITypes.Rollup,
title: ncName.title,
column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId,
fk_rollup_column_id: ncRollupColumnId,
rollup_function: ncRollupFn,
},
req: {
user: syncDB.user.email,
clientIp: '',
},
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create'); recordPerfStats(_perfStart, 'dbTableColumn.create');
@ -1261,12 +1394,26 @@ export default async (
logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, { // const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Lookup, // uidt: UITypes.Lookup,
title: ncName.title, // title: ncName.title,
column_name: ncName.column_name, // column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId, // fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId, // fk_lookup_column_id: ncLookupColumnId,
// });
const ncTbl: any = await columnService.columnAdd({
tableId: srcTableId,
column: {
uidt: UITypes.Lookup,
title: ncName.title,
column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId,
},
req: {
user: syncDB.user.email,
clientIp: '',
},
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create'); recordPerfStats(_perfStart, 'dbTableColumn.create');
@ -1302,7 +1449,8 @@ export default async (
if (ncColId) { if (ncColId) {
logDetailed(`NC API: dbTableColumn.primaryColumnSet`); logDetailed(`NC API: dbTableColumn.primaryColumnSet`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
await api.dbTableColumn.primaryColumnSet(ncColId); // await api.dbTableColumn.primaryColumnSet(ncColId);
await columnService.columnSetAsPrimary({ columnId: ncColId });
recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet'); recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet');
// update schema // update schema
@ -1319,13 +1467,19 @@ export default async (
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
if (viewType === 'form') { if (viewType === 'form') {
viewDetails = (await api.dbView.formRead(viewId)).columns; // viewDetails = (await api.dbView.formRead(viewId)).columns;
viewDetails = (await formViewService.formViewGet({ formViewId: viewId }))
.columns;
recordPerfStats(_perfStart, 'dbView.formRead'); recordPerfStats(_perfStart, 'dbView.formRead');
} else if (viewType === 'gallery') { } else if (viewType === 'gallery') {
viewDetails = (await api.dbView.galleryRead(viewId)).columns; // viewDetails = (await api.dbView.galleryRead(viewId)).columns;
viewDetails = (
await galleryViewService.galleryViewGet({ galleryViewId: viewId })
).columns;
recordPerfStats(_perfStart, 'dbView.galleryRead'); recordPerfStats(_perfStart, 'dbView.galleryRead');
} else { } else {
viewDetails = await api.dbView.gridColumnsList(viewId); // viewDetails = await api.dbView.gridColumnsList(viewId);
viewDetails = await viewColumnService.columnList({ viewId: viewId });
recordPerfStats(_perfStart, 'dbView.gridColumnsList'); recordPerfStats(_perfStart, 'dbView.gridColumnsList');
} }
@ -1453,17 +1607,26 @@ export default async (
?.map((a) => a.filename?.split('?')?.[0]) ?.map((a) => a.filename?.split('?')?.[0])
.join(', ')}` .join(', ')}`
); );
tempArr = await api.storage.uploadByUrl( // tempArr = await api.storage.uploadByUrl(
{ // {
path: `noco/${sDB.projectName}/${table.title}/${key}`, // path: `noco/${sDB.projectName}/${table.title}/${key}`,
}, // },
value?.map((attachment) => ({ // value?.map((attachment) => ({
// fileName: attachment.filename?.split('?')?.[0],
// url: attachment.url,
// size: attachment.size,
// mimetype: attachment.type,
// }))
// );
tempArr = await attachmentService.uploadViaURL({
path: `noco/${sDB.projectName}/${table.title}/${key}`,
urls: value?.map((attachment) => ({
fileName: attachment.filename?.split('?')?.[0], fileName: attachment.filename?.split('?')?.[0],
url: attachment.url, url: attachment.url,
size: attachment.size, size: attachment.size,
mimetype: attachment.type, mimetype: attachment.type,
})) })),
); });
} catch (e) { } catch (e) {
console.log(e); console.log(e);
} }
@ -1543,9 +1706,15 @@ export default async (
// create empty project (XC-DB) // create empty project (XC-DB)
logDetailed(`Create Project: ${projName}`); logDetailed(`Create Project: ${projName}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
ncCreatedProjectSchema = await api.project.create({ // ncCreatedProjectSchema = await api.project.create({
title: projName, // title: projName,
// });
ncCreatedProjectSchema = await projectService.projectCreate({
project: { title: projName },
user: { id: syncDB.user.id },
}); });
recordPerfStats(_perfStart, 'project.create'); recordPerfStats(_perfStart, 'project.create');
} }
@ -1553,7 +1722,10 @@ export default async (
// create empty project (XC-DB) // create empty project (XC-DB)
logDetailed(`Getting project meta: ${projId}`); logDetailed(`Getting project meta: ${projId}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
ncCreatedProjectSchema = await api.project.read(projId); // ncCreatedProjectSchema = await api.project.read(projId);
ncCreatedProjectSchema = await projectService.getProjectWithInfo({
projectId: projId,
});
recordPerfStats(_perfStart, 'project.read'); recordPerfStats(_perfStart, 'project.read');
} }
@ -1585,7 +1757,13 @@ export default async (
logDetailed(`NC API dbView.galleryCreate :: ${viewName}`); logDetailed(`NC API dbView.galleryCreate :: ${viewName}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
await api.dbView.galleryCreate(tblId, { title: viewName }); // await api.dbView.galleryCreate(tblId, { title: viewName });
await galleryViewService.galleryViewCreate({
tableId: tblId,
gallery: {
title: viewName,
},
});
recordPerfStats(_perfStart, 'dbView.galleryCreate'); recordPerfStats(_perfStart, 'dbView.galleryCreate');
await updateNcTblSchemaById(tblId); await updateNcTblSchemaById(tblId);
@ -1646,7 +1824,11 @@ export default async (
logDetailed(`NC API dbView.formCreate :: ${viewName}`); logDetailed(`NC API dbView.formCreate :: ${viewName}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const f = await api.dbView.formCreate(tblId, formData); // const f = await api.dbView.formCreate(tblId, formData);
const f = await formViewService.formViewCreate({
tableId: tblId,
body: formData,
});
recordPerfStats(_perfStart, 'dbView.formCreate'); recordPerfStats(_perfStart, 'dbView.formCreate');
logDetailed( logDetailed(
@ -1689,7 +1871,12 @@ export default async (
(x) => x.id === gridViews[i].id (x) => x.id === gridViews[i].id
)?.name; )?.name;
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const viewList: any = await api.dbView.list(tblId); // const viewList: any = await api.dbView.list(tblId);
const viewList = { list: [] };
viewList['list'] = await viewService.viewList({
tableId: tblId,
user: { roles: userRole },
});
recordPerfStats(_perfStart, 'dbView.list'); recordPerfStats(_perfStart, 'dbView.list');
let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id; let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id;
@ -1704,8 +1891,14 @@ export default async (
if (i > 0) { if (i > 0) {
logDetailed(`NC API dbView.gridCreate :: ${viewName}`); logDetailed(`NC API dbView.gridCreate :: ${viewName}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const viewCreated = await api.dbView.gridCreate(tblId, { // const viewCreated = await api.dbView.gridCreate(tblId, {
title: viewName, // title: viewName,
// });
const viewCreated = await gridViewService.gridViewCreate({
tableId: tblId,
grid: {
title: viewName,
},
}); });
recordPerfStats(_perfStart, 'dbView.gridCreate'); recordPerfStats(_perfStart, 'dbView.gridCreate');
@ -1773,10 +1966,19 @@ export default async (
); );
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
insertJobs.push( insertJobs.push(
api.auth // api.auth
.projectUserAdd(ncCreatedProjectSchema.id, { // .projectUserAdd(ncCreatedProjectSchema.id, {
email: value.email, // email: value.email,
roles: userRoles[value.permissionLevel], // roles: userRoles[value.permissionLevel],
// })
projectUserService
.userInvite({
projectId: ncCreatedProjectSchema.id,
projectUser: {
email: value.email,
roles: userRoles[value.permissionLevel],
},
req: { user: syncDB.user, clientIp: '' },
}) })
.catch((e) => .catch((e) =>
e.response?.data?.msg e.response?.data?.msg
@ -1803,7 +2005,11 @@ export default async (
async function updateNcTblSchemaById(tblId) { async function updateNcTblSchemaById(tblId) {
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const ncTbl = await api.dbTable.read(tblId); // const ncTbl = await api.dbTable.read(tblId);
const ncTbl: any = await tableService.getTableWithAccessibleViews({
tableId: tblId,
user: syncDB.user,
});
recordPerfStats(_perfStart, 'dbTable.read'); recordPerfStats(_perfStart, 'dbTable.read');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
@ -2060,8 +2266,12 @@ export default async (
// insert filters // insert filters
for (let i = 0; i < ncFilters.length; i++) { for (let i = 0; i < ncFilters.length; i++) {
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
await api.dbTableFilter.create(viewId, { // await api.dbTableFilter.create(viewId, {
...ncFilters[i], // ...ncFilters[i],
// });
await filterService.filterCreate({
viewId: viewId,
filter: ncFilters[i],
}); });
recordPerfStats(_perfStart, 'dbTableFilter.create'); recordPerfStats(_perfStart, 'dbTableFilter.create');
@ -2076,9 +2286,16 @@ export default async (
if (columnId) { if (columnId) {
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
await api.dbTableSort.create(viewId, { // await api.dbTableSort.create(viewId, {
fk_column_id: columnId, // fk_column_id: columnId,
direction: s.sortSet[i].ascending ? 'asc' : 'desc', // direction: s.sortSet[i].ascending ? 'asc' : 'desc',
// });
await sortService.sortCreate({
viewId: viewId,
sort: {
fk_column_id: columnId,
direction: s.sortSet[i].ascending ? 'asc' : 'desc',
},
}); });
recordPerfStats(_perfStart, 'dbTableSort.create'); recordPerfStats(_perfStart, 'dbTableSort.create');
} }
@ -2100,13 +2317,21 @@ export default async (
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
if (viewType === 'form') { if (viewType === 'form') {
viewDetails = (await api.dbView.formRead(viewId)).columns; // viewDetails = (await api.dbView.formRead(viewId)).columns;
viewDetails = (await formViewService.formViewGet({ formViewId: viewId }))
.columns;
recordPerfStats(_perfStart, 'dbView.formRead'); recordPerfStats(_perfStart, 'dbView.formRead');
} else if (viewType === 'gallery') { } else if (viewType === 'gallery') {
viewDetails = (await api.dbView.galleryRead(viewId)).columns; // viewDetails = (await api.dbView.galleryRead(viewId)).columns;
viewDetails = (
await galleryViewService.galleryViewGet({
galleryViewId: viewId,
})
).columns;
recordPerfStats(_perfStart, 'dbView.galleryRead'); recordPerfStats(_perfStart, 'dbView.galleryRead');
} else { } else {
viewDetails = await api.dbView.gridColumnsList(viewId); // viewDetails = await api.dbView.gridColumnsList(viewId);
viewDetails = await viewColumnService.columnList({ viewId: viewId });
recordPerfStats(_perfStart, 'dbView.gridColumnsList'); recordPerfStats(_perfStart, 'dbView.gridColumnsList');
} }
@ -2127,9 +2352,17 @@ export default async (
// first two positions held by record id & record hash // first two positions held by record id & record hash
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
await api.dbViewColumn.update(viewId, ncViewColumnId, { // await api.dbViewColumn.update(viewId, ncViewColumnId, {
show: false, // show: false,
order: j + 1 + c.length, // order: j + 1 + c.length,
// });
await viewColumnService.columnUpdate({
viewId: viewId,
columnId: ncViewColumnId,
column: {
show: false,
order: j + 1 + c.length,
},
}); });
recordPerfStats(_perfStart, 'dbViewColumn.update'); recordPerfStats(_perfStart, 'dbViewColumn.update');
} }
@ -2154,12 +2387,21 @@ export default async (
if (x?.required) formData[`required`] = x.required; if (x?.required) formData[`required`] = x.required;
if (x?.description) formData[`description`] = x.description; if (x?.description) formData[`description`] = x.description;
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
await api.dbView.formColumnUpdate(ncViewColumnId, formData); // await api.dbView.formColumnUpdate(ncViewColumnId, formData);
await formViewColumnService.columnUpdate({
formViewColumnId: ncViewColumnId,
formViewColumn: formData,
});
recordPerfStats(_perfStart, 'dbView.formColumnUpdate'); recordPerfStats(_perfStart, 'dbView.formColumnUpdate');
} }
} }
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
await api.dbViewColumn.update(viewId, ncViewColumnId, configData); // await api.dbViewColumn.update(viewId, ncViewColumnId, configData);
await viewColumnService.columnUpdate({
viewId: viewId,
columnId: ncViewColumnId,
column: configData,
});
recordPerfStats(_perfStart, 'dbViewColumn.update'); recordPerfStats(_perfStart, 'dbViewColumn.update');
} }
} }
@ -2168,13 +2410,6 @@ export default async (
let recordCnt = 0; let recordCnt = 0;
try { try {
logBasic('SDK initialized'); logBasic('SDK initialized');
api = new Api({
baseURL: syncDB.baseURL,
headers: {
'xc-auth': syncDB.authToken,
},
});
logDetailed('Project initialization started'); logDetailed('Project initialization started');
// delete project if already exists // delete project if already exists
if (debugMode) await init(syncDB); if (debugMode) await init(syncDB);
@ -2254,10 +2489,16 @@ export default async (
try { try {
// await nc_DumpTableSchema(); // await nc_DumpTableSchema();
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const ncTblList = await api.base.tableList( // const ncTblList = await api.base.tableList(
ncCreatedProjectSchema.id, // ncCreatedProjectSchema.id,
syncDB.baseId // syncDB.baseId
); // );
const ncTblList = { list: [] };
ncTblList['list'] = await tableService.getAccessibleTables({
projectId: ncCreatedProjectSchema.id,
baseId: syncDB.baseId,
roles: userRole,
});
recordPerfStats(_perfStart, 'base.tableList'); recordPerfStats(_perfStart, 'base.tableList');
logBasic('Reading Records...'); logBasic('Reading Records...');
@ -2273,7 +2514,11 @@ export default async (
continue; continue;
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const ncTbl = await api.dbTable.read(ncTblList.list[i].id); // const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
const ncTbl: any = await tableService.getTableWithAccessibleViews({
tableId: ncTblList.list[i].id,
user: syncDB.user,
});
recordPerfStats(_perfStart, 'dbTable.read'); recordPerfStats(_perfStart, 'dbTable.read');
recordCnt = 0; recordCnt = 0;
@ -2283,7 +2528,6 @@ export default async (
projectName: syncDB.projectName, projectName: syncDB.projectName,
table: ncTbl, table: ncTbl,
base, base,
api,
logBasic, logBasic,
nocoBaseDataProcessing_v2, nocoBaseDataProcessing_v2,
sDB: syncDB, sDB: syncDB,
@ -2303,12 +2547,15 @@ export default async (
) )
continue; continue;
const ncTbl = await api.dbTable.read(ncTblList.list[i].id); // const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
const ncTbl: any = await tableService.getTableWithAccessibleViews({
tableId: ncTblList.list[i].id,
user: syncDB.user,
});
rtc.data.nestedLinks += await importLTARData({ rtc.data.nestedLinks += await importLTARData({
table: ncTbl, table: ncTbl,
projectName: syncDB.projectName, projectName: syncDB.projectName,
api,
base, base,
fields: null, //Object.values(tblLinkGroup).flat(), fields: null, //Object.values(tblLinkGroup).flat(),
logBasic, logBasic,
@ -2317,6 +2564,7 @@ export default async (
records: recordsMap[ncTbl.id], records: recordsMap[ncTbl.id],
atNcAliasRef, atNcAliasRef,
ncLinkMappingTable, ncLinkMappingTable,
syncDB,
}); });
} }
@ -2417,6 +2665,7 @@ export interface AirtableSyncConfig {
baseId?: string; baseId?: string;
apiKey: string; apiKey: string;
shareId: string; shareId: string;
user: UserType;
options: { options: {
syncViews: boolean; syncViews: boolean;
syncData: boolean; syncData: boolean;

105
packages/nocodb/src/lib/services/sync/helpers/readAndProcessData.ts
Unescape View File

@ -1,7 +1,8 @@
import { RelationTypes, UITypes } from 'nocodb-sdk'; import { RelationTypes, UITypes } from 'nocodb-sdk';
import { bulkDataService, tableService } from '../..';
import EntityMap from './EntityMap'; import EntityMap from './EntityMap';
import type { AirtableBase } from 'airtable/lib/airtable_base'; import type { AirtableBase } from 'airtable/lib/airtable_base';
import type { Api, TableType } from 'nocodb-sdk'; import type { TableType } from 'nocodb-sdk';
const BULK_DATA_BATCH_SIZE = 500; const BULK_DATA_BATCH_SIZE = 500;
const ASSOC_BULK_DATA_BATCH_SIZE = 1000; const ASSOC_BULK_DATA_BATCH_SIZE = 1000;
@ -70,7 +71,6 @@ export async function importData({
projectName, projectName,
table, table,
base, base,
api,
nocoBaseDataProcessing_v2, nocoBaseDataProcessing_v2,
sDB, sDB,
logDetailed = (_str) => {}, logDetailed = (_str) => {},
@ -82,7 +82,6 @@ export async function importData({
base: AirtableBase; base: AirtableBase;
logBasic: (string) => void; logBasic: (string) => void;
logDetailed: (string) => void; logDetailed: (string) => void;
api: Api<any>;
nocoBaseDataProcessing_v2; nocoBaseDataProcessing_v2;
sDB; sDB;
}): Promise<EntityMap> { }): Promise<EntityMap> {
@ -116,12 +115,20 @@ export async function importData({
if (tempData.length >= BULK_DATA_BATCH_SIZE) { if (tempData.length >= BULK_DATA_BATCH_SIZE) {
let insertArray = tempData.splice(0, tempData.length); let insertArray = tempData.splice(0, tempData.length);
await api.dbTableRow.bulkCreate( // await api.dbTableRow.bulkCreate(
'nc', // 'nc',
projectName, // projectName,
table.id, // table.id,
insertArray // insertArray
); // );
await bulkDataService.bulkDataInsert({
projectName: projectName,
tableName: table.title,
body: insertArray,
cookie: {},
});
logBasic( logBasic(
`:: Importing '${ `:: Importing '${
table.title table.title
@ -142,12 +149,20 @@ export async function importData({
readable.on('end', async () => { readable.on('end', async () => {
await Promise.all(promises); await Promise.all(promises);
if (tempData.length > 0) { if (tempData.length > 0) {
await api.dbTableRow.bulkCreate( // await api.dbTableRow.bulkCreate(
'nc', // 'nc',
projectName, // projectName,
table.id, // table.id,
tempData // tempData
); // );
await bulkDataService.bulkDataInsert({
projectName: projectName,
tableName: table.title,
body: tempData,
cookie: {},
});
logBasic( logBasic(
`:: Importing '${ `:: Importing '${
table.title table.title
@ -174,7 +189,6 @@ export async function importLTARData({
table, table,
fields, fields,
base, base,
api,
projectName, projectName,
insertedAssocRef = {}, insertedAssocRef = {},
logDetailed = (_str) => {}, logDetailed = (_str) => {},
@ -182,6 +196,7 @@ export async function importLTARData({
records, records,
atNcAliasRef, atNcAliasRef,
ncLinkMappingTable, ncLinkMappingTable,
syncDB,
}: { }: {
projectName: string; projectName: string;
table: { title?: string; id?: string }; table: { title?: string; id?: string };
@ -189,7 +204,6 @@ export async function importLTARData({
base: AirtableBase; base: AirtableBase;
logDetailed: (string) => void; logDetailed: (string) => void;
logBasic: (string) => void; logBasic: (string) => void;
api: Api<any>;
insertedAssocRef: { [assocTableId: string]: boolean }; insertedAssocRef: { [assocTableId: string]: boolean };
records?: EntityMap; records?: EntityMap;
atNcAliasRef: { atNcAliasRef: {
@ -198,6 +212,7 @@ export async function importLTARData({
}; };
}; };
ncLinkMappingTable: Record<string, Record<string, any>>[]; ncLinkMappingTable: Record<string, Record<string, any>>[];
syncDB;
}) { }) {
const assocTableMetas: Array<{ const assocTableMetas: Array<{
modelMeta: { id?: string; title?: string }; modelMeta: { id?: string; title?: string };
@ -215,7 +230,11 @@ export async function importLTARData({
logBasic, logBasic,
})); }));
const modelMeta: any = await api.dbTable.read(table.id); // const modelMeta: any = await api.dbTable.read(table.id);
const modelMeta: any = await tableService.getTableWithAccessibleViews({
tableId: table.id,
user: syncDB.user,
});
for (const colMeta of modelMeta.columns) { for (const colMeta of modelMeta.columns) {
// skip columns which are not LTAR and Many to many // skip columns which are not LTAR and Many to many
@ -235,9 +254,15 @@ export async function importLTARData({
// mark as inserted // mark as inserted
insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true;
const assocModelMeta: TableType = (await api.dbTable.read( // const assocModelMeta: TableType = (await api.dbTable.read(
colMeta.colOptions.fk_mm_model_id // colMeta.colOptions.fk_mm_model_id
)) as any; // )) as any;
const assocModelMeta: TableType =
(await tableService.getTableWithAccessibleViews({
tableId: colMeta.colOptions.fk_mm_model_id,
user: syncDB.user,
})) as any;
// extract associative table and columns meta // extract associative table and columns meta
assocTableMetas.push({ assocTableMetas.push({
@ -291,12 +316,19 @@ export async function importLTARData({
)}` )}`
); );
await api.dbTableRow.bulkCreate( // await api.dbTableRow.bulkCreate(
'nc', // 'nc',
projectName, // projectName,
assocMeta.modelMeta.id, // assocMeta.modelMeta.id,
insertArray // insertArray
); // );
await bulkDataService.bulkDataInsert({
projectName: projectName,
tableName: table.title,
body: insertArray,
cookie: {},
});
importedCount += insertArray.length; importedCount += insertArray.length;
insertArray = []; insertArray = [];
@ -319,12 +351,19 @@ export async function importLTARData({
)}` )}`
); );
await api.dbTableRow.bulkCreate( // await api.dbTableRow.bulkCreate(
'nc', // 'nc',
projectName, // projectName,
assocMeta.modelMeta.id, // assocMeta.modelMeta.id,
assocTableData // assocTableData
); // );
await bulkDataService.bulkDataInsert({
projectName: projectName,
tableName: table.title,
body: assocTableData,
cookie: {},
});
importedCount += assocTableData.length; importedCount += assocTableData.length;
assocTableData = []; assocTableData = [];

Write Preview
Loading…
Cancel
Save