github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: use dynamic entity map for AT import 

Signed-off-by: mertmit <mertmit99@gmail.com>
pull/4156/head
mertmit 2 years ago
parent
3a0ba79eb5
commit
d69337e8e8
3 changed files with 182 additions and 128 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 18
      packages/nocodb/src/lib/meta/api/sync/helpers/EntityMap.ts
  2. 86
      packages/nocodb/src/lib/meta/api/sync/helpers/job.ts
  3. 206
      packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts

18
packages/nocodb/src/lib/meta/api/sync/helpers/EntityMap.ts
Unescape View File

@ -92,7 +92,11 @@ class EntityMap {
if (rs) { if (rs) {
for (const key of Object.keys(rs)) { for (const key of Object.keys(rs)) {
if (rs[key] && rs[key].startsWith('JSON::')) { if (rs[key] && rs[key].startsWith('JSON::')) {
rs[key] = JSON.parse(rs[key].replace('JSON::', '')); try {
rs[key] = JSON.parse(rs[key].replace('JSON::', ''));
} catch (e) {
console.log(e);
}
} }
} }
} }
@ -136,7 +140,11 @@ class EntityMap {
for (const row of rs) { for (const row of rs) {
for (const key of Object.keys(row)) { for (const key of Object.keys(row)) {
if (row[key] && row[key].startsWith('JSON::')) { if (row[key] && row[key].startsWith('JSON::')) {
row[key] = JSON.parse(row[key].replace('JSON::', '')); try {
row[key] = JSON.parse(row[key].replace('JSON::', ''));
} catch (e) {
console.log(e);
}
} }
} }
} }
@ -168,7 +176,11 @@ class DBStream extends Readable {
if (result) { if (result) {
for (const key of Object.keys(result)) { for (const key of Object.keys(result)) {
if (result[key] && result[key].startsWith('JSON::')) { if (result[key] && result[key].startsWith('JSON::')) {
result[key] = JSON.parse(result[key].replace('JSON::', '')); try {
result[key] = JSON.parse(result[key].replace('JSON::', ''));
} catch (e) {
console.log(e);
}
} }
} }
} }

86
packages/nocodb/src/lib/meta/api/sync/helpers/job.ts
Unescape View File

@ -14,6 +14,8 @@ import utc from 'dayjs/plugin/utc';
import tinycolor from 'tinycolor2'; import tinycolor from 'tinycolor2';
import { importData, importLTARData } from './readAndProcessData'; import { importData, importLTARData } from './readAndProcessData';
import EntityMap from './EntityMap';
dayjs.extend(utc); dayjs.extend(utc);
const selectColors = { const selectColors = {
@ -67,32 +69,28 @@ export default async (
syncDB: AirtableSyncConfig, syncDB: AirtableSyncConfig,
progress: (data: { msg?: string; level?: any }) => void progress: (data: { msg?: string; level?: any }) => void
) => { ) => {
const sMap = { const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent');
mapTbl: {}, await sMapEM.init();
const sMap = {
// static mapping records between aTblId && ncId // static mapping records between aTblId && ncId
addToMappingTbl(aTblId, ncId, ncName, parent?) { async addToMappingTbl(aTblId, ncId, ncName, ncParent?) {
this.mapTbl[aTblId] = { await sMapEM.addRow({ aTblId, ncId, ncName, ncParent });
ncId: ncId,
ncParent: parent,
// name added to assist in quick debug
ncName: ncName,
};
}, },
// get NcID from airtable ID // get NcID from airtable ID
getNcIdFromAtId(aId) { async getNcIdFromAtId(aId) {
return this.mapTbl[aId]?.ncId; return (await sMapEM.getRow('aTblId', aId, ['ncId']))?.ncId;
}, },
// get nc Parent from airtable ID // get nc Parent from airtable ID
getNcParentFromAtId(aId) { async getNcParentFromAtId(aId) {
return this.mapTbl[aId]?.ncParent; return (await sMapEM.getRow('aTblId', aId, ['ncParent']))?.ncParent;
}, },
// get nc-title from airtable ID // get nc-title from airtable ID
getNcNameFromAtId(aId) { async getNcNameFromAtId(aId) {
return this.mapTbl[aId]?.ncName; return (await sMapEM.getRow('aTblId', aId, ['ncName']))?.ncName;
}, },
}; };
@ -333,8 +331,8 @@ export default async (
// let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn); // let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn);
// return ncCol; // return ncCol;
const ncTblId = sMap.getNcParentFromAtId(aTblFieldId); const ncTblId = await sMap.getNcParentFromAtId(aTblFieldId);
const ncColId = sMap.getNcIdFromAtId(aTblFieldId); const ncColId = await sMap.getNcIdFromAtId(aTblFieldId);
// not migrated column, skip // not migrated column, skip
if (ncColId === undefined || ncTblId === undefined) return 0; if (ncColId === undefined || ncTblId === undefined) return 0;
@ -424,7 +422,7 @@ export default async (
// retrieve additional options associated with selected data types // retrieve additional options associated with selected data types
// //
function getNocoTypeOptions(col: any): any { async function getNocoTypeOptions(col: any): Promise<any> {
switch (col.type) { switch (col.type) {
case 'select': case 'select':
case 'multiSelect': { case 'multiSelect': {
@ -457,7 +455,7 @@ export default async (
: tinycolor.random().toHexString(), : tinycolor.random().toHexString(),
}); });
sMap.addToMappingTbl( await sMap.addToMappingTbl(
(value as any).id, (value as any).id,
undefined, undefined,
(value as any).name (value as any).name
@ -472,7 +470,7 @@ export default async (
// convert to Nc schema (basic, excluding relations) // convert to Nc schema (basic, excluding relations)
// //
function tablesPrepare(tblSchema: any[]) { async function tablesPrepare(tblSchema: any[]) {
const tables: any[] = []; const tables: any[] = [];
for (let i = 0; i < tblSchema.length; ++i) { for (let i = 0; i < tblSchema.length; ++i) {
@ -569,7 +567,7 @@ export default async (
} }
// additional column parameters when applicable // additional column parameters when applicable
const colOptions = getNocoTypeOptions(col); const colOptions = await getNocoTypeOptions(col);
switch (colOptions.type) { switch (colOptions.type) {
case 'select': case 'select':
@ -602,7 +600,7 @@ export default async (
async function nocoCreateBaseSchema(aTblSchema) { async function nocoCreateBaseSchema(aTblSchema) {
// base schema preparation: exclude // base schema preparation: exclude
const tables: any[] = tablesPrepare(aTblSchema); const tables: any[] = await tablesPrepare(aTblSchema);
// for each table schema, create nc table // for each table schema, create nc table
for (let idx = 0; idx < tables.length; idx++) { for (let idx = 0; idx < tables.length; idx++) {
@ -696,7 +694,7 @@ export default async (
if (!nc_isLinkExists(aTblLinkColumns[i].id)) { if (!nc_isLinkExists(aTblLinkColumns[i].id)) {
// parent table ID // parent table ID
// let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
const srcTableId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
// find child table name from symmetric column ID specified // find child table name from symmetric column ID specified
// self link, symmetricColumnId field will be undefined // self link, symmetricColumnId field will be undefined
@ -911,7 +909,7 @@ export default async (
// parent table ID // parent table ID
// let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
const srcTableId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
const srcTableSchema = ncSchema.tablesById[srcTableId]; const srcTableSchema = ncSchema.tablesById[srcTableId];
if (aTblColumns.length) { if (aTblColumns.length) {
@ -939,10 +937,10 @@ export default async (
continue; continue;
} }
const ncRelationColumnId = sMap.getNcIdFromAtId( const ncRelationColumnId = await sMap.getNcIdFromAtId(
aTblColumns[i].typeOptions.relationColumnId aTblColumns[i].typeOptions.relationColumnId
); );
const ncLookupColumnId = sMap.getNcIdFromAtId( const ncLookupColumnId = await sMap.getNcIdFromAtId(
aTblColumns[i].typeOptions.foreignTableRollupColumnId aTblColumns[i].typeOptions.foreignTableRollupColumnId
); );
@ -1015,10 +1013,10 @@ export default async (
const srcTableId = nestedLookupTbl[0].srcTableId; const srcTableId = nestedLookupTbl[0].srcTableId;
const srcTableSchema = ncSchema.tablesById[srcTableId]; const srcTableSchema = ncSchema.tablesById[srcTableId];
const ncRelationColumnId = sMap.getNcIdFromAtId( const ncRelationColumnId = await sMap.getNcIdFromAtId(
nestedLookupTbl[0].typeOptions.relationColumnId nestedLookupTbl[0].typeOptions.relationColumnId
); );
const ncLookupColumnId = sMap.getNcIdFromAtId( const ncLookupColumnId = await sMap.getNcIdFromAtId(
nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId
); );
@ -1101,7 +1099,7 @@ export default async (
// parent table ID // parent table ID
// let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
const srcTableId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
const srcTableSchema = ncSchema.tablesById[srcTableId]; const srcTableSchema = ncSchema.tablesById[srcTableId];
if (aTblColumns.length) { if (aTblColumns.length) {
@ -1146,10 +1144,10 @@ export default async (
continue; continue;
} }
const ncRelationColumnId = sMap.getNcIdFromAtId( const ncRelationColumnId = await sMap.getNcIdFromAtId(
aTblColumns[i].typeOptions.relationColumnId aTblColumns[i].typeOptions.relationColumnId
); );
const ncRollupColumnId = sMap.getNcIdFromAtId( const ncRollupColumnId = await sMap.getNcIdFromAtId(
aTblColumns[i].typeOptions.foreignTableRollupColumnId aTblColumns[i].typeOptions.foreignTableRollupColumnId
); );
@ -1219,10 +1217,10 @@ export default async (
const srcTableId = nestedLookupTbl[0].srcTableId; const srcTableId = nestedLookupTbl[0].srcTableId;
const srcTableSchema = ncSchema.tablesById[srcTableId]; const srcTableSchema = ncSchema.tablesById[srcTableId];
const ncRelationColumnId = sMap.getNcIdFromAtId( const ncRelationColumnId = await sMap.getNcIdFromAtId(
nestedLookupTbl[0].typeOptions.relationColumnId nestedLookupTbl[0].typeOptions.relationColumnId
); );
const ncLookupColumnId = sMap.getNcIdFromAtId( const ncLookupColumnId = await sMap.getNcIdFromAtId(
nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId
); );
@ -1278,7 +1276,7 @@ export default async (
); );
const pColId = aTblSchema[idx].primaryColumnId; const pColId = aTblSchema[idx].primaryColumnId;
const ncColId = sMap.getNcIdFromAtId(pColId); const ncColId = await sMap.getNcIdFromAtId(pColId);
// skip primary column configuration if we field not migrated // skip primary column configuration if we field not migrated
if (ncColId) { if (ncColId) {
@ -1288,7 +1286,7 @@ export default async (
recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet'); recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet');
// update schema // update schema
const ncTblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const ncTblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
await updateNcTblSchemaById(ncTblId); await updateNcTblSchemaById(ncTblId);
} }
} }
@ -1408,7 +1406,7 @@ export default async (
case UITypes.MultiSelect: case UITypes.MultiSelect:
rec[key] = value rec[key] = value
.map((v) => { ?.map((v) => {
if (v === '') { if (v === '') {
return 'nc_empty'; return 'nc_empty';
} }
@ -1567,7 +1565,7 @@ export default async (
async function nocoConfigureFormView(sDB, aTblSchema) { async function nocoConfigureFormView(sDB, aTblSchema) {
if (!sDB.options.syncViews) return; if (!sDB.options.syncViews) return;
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
const formViews = aTblSchema[idx].views.filter((x) => x.type === 'form'); const formViews = aTblSchema[idx].views.filter((x) => x.type === 'form');
const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form;
@ -1639,7 +1637,7 @@ export default async (
async function nocoConfigureGridView(sDB, aTblSchema) { async function nocoConfigureGridView(sDB, aTblSchema) {
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
const gridViews = aTblSchema[idx].views.filter((x) => x.type === 'grid'); const gridViews = aTblSchema[idx].views.filter((x) => x.type === 'grid');
let viewCnt = idx; let viewCnt = idx;
@ -1955,7 +1953,7 @@ export default async (
// one of not migrated column; // one of not migrated column;
if (!colSchema) { if (!colSchema) {
updateMigrationSkipLog( updateMigrationSkipLog(
sMap.getNcNameFromAtId(viewId), await sMap.getNcNameFromAtId(viewId),
colSchema.title, colSchema.title,
colSchema.uidt, colSchema.uidt,
`filter config skipped; column not migrated` `filter config skipped; column not migrated`
@ -1970,7 +1968,7 @@ export default async (
if (datatype === UITypes.Date || datatype === UITypes.DateTime) { if (datatype === UITypes.Date || datatype === UITypes.DateTime) {
// skip filters over data datatype // skip filters over data datatype
updateMigrationSkipLog( updateMigrationSkipLog(
sMap.getNcNameFromAtId(viewId), await sMap.getNcNameFromAtId(viewId),
colSchema.title, colSchema.title,
colSchema.uidt, colSchema.uidt,
`filter config skipped; filter over date datatype not supported` `filter config skipped; filter over date datatype not supported`
@ -1990,7 +1988,7 @@ export default async (
fk_column_id: columnId, fk_column_id: columnId,
logical_op: f.conjunction, logical_op: f.conjunction,
comparison_op: filterMap[filter.operator], comparison_op: filterMap[filter.operator],
value: sMap.getNcNameFromAtId(filter.value[i]), value: await sMap.getNcNameFromAtId(filter.value[i]),
}; };
ncFilters.push(fx); ncFilters.push(fx);
} }
@ -2001,7 +1999,7 @@ export default async (
fk_column_id: columnId, fk_column_id: columnId,
logical_op: f.conjunction, logical_op: f.conjunction,
comparison_op: filterMap[filter.operator], comparison_op: filterMap[filter.operator],
value: sMap.getNcNameFromAtId(filter.value), value: await sMap.getNcNameFromAtId(filter.value),
}; };
ncFilters.push(fx); ncFilters.push(fx);
} }
@ -2097,7 +2095,7 @@ export default async (
// rest of the columns from airtable- retain order & visibility property // rest of the columns from airtable- retain order & visibility property
for (let j = 0; j < c.length; j++) { for (let j = 0; j < c.length; j++) {
const ncColumnId = sMap.getNcIdFromAtId(c[j].columnId); const ncColumnId = await sMap.getNcIdFromAtId(c[j].columnId);
const ncViewColumnId = await nc_getViewColumnId( const ncViewColumnId = await nc_getViewColumnId(
viewId, viewId,
viewType, viewType,
@ -2243,7 +2241,7 @@ export default async (
sDB: syncDB, sDB: syncDB,
logDetailed, logDetailed,
}); });
rtc.data.records += recordsMap[ncTbl.id].length; rtc.data.records += await recordsMap[ncTbl.id].getCount();
logDetailed(`Data inserted from ${ncTbl.title}`); logDetailed(`Data inserted from ${ncTbl.title}`);
} }

206
packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts
Unescape View File

@ -1,59 +1,54 @@
import { AirtableBase } from 'airtable/lib/airtable_base'; import { AirtableBase } from 'airtable/lib/airtable_base';
import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk'; import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk';
import EntityMap from './EntityMap';
const BULK_DATA_BATCH_SIZE = 2000;
const ASSOC_BULK_DATA_BATCH_SIZE = 5000; const BULK_DATA_BATCH_SIZE = 500;
const ASSOC_BULK_DATA_BATCH_SIZE = 2000;
async function readAllData({ async function readAllData({
table, table,
fields, fields,
base, base,
logBasic = (_str) => {}, logBasic = (_str) => {},
triggerThreshold = BULK_DATA_BATCH_SIZE,
onThreshold = async (_rec) => {},
}: { }: {
table: { title?: string }; table: { title?: string };
fields?; fields?;
base: AirtableBase; base: AirtableBase;
logBasic?: (string) => void; logBasic?: (string) => void;
logDetailed?: (string) => void; logDetailed?: (string) => void;
triggerThreshold?: number; }): Promise<EntityMap> {
onThreshold?: (
records: Array<{ fields: any; id: string }>,
allRecords?: Array<{ fields: any; id: string }>
) => Promise<void>;
}): Promise<Array<any>> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const data = []; let data = null;
let thresholdCbkData = [];
const selectParams: any = { const selectParams: any = {
pageSize: 100, pageSize: 100,
}; };
if (fields) selectParams.fields = fields; if (fields) selectParams.fields = fields;
const insertJobs: Promise<any>[] = [];
base(table.title) base(table.title)
.select(selectParams) .select(selectParams)
.eachPage( .eachPage(
async function page(records, fetchNextPage) { async function page(records, fetchNextPage) {
data.push(...records); if (!data) {
thresholdCbkData.push(...records); data = new EntityMap();
await data.init();
}
for await (const record of records) {
await data.addRow({ id: record.id, ...record.fields });
}
const tmpLength = await data.getCount();
logBasic( logBasic(
`:: Reading '${table.title}' data :: ${Math.max( `:: Reading '${table.title}' data :: ${Math.max(
1, 1,
data.length - records.length tmpLength - records.length
)} - ${data.length}` )} - ${tmpLength}`
); );
if (thresholdCbkData.length >= triggerThreshold) {
await Promise.all(insertJobs);
insertJobs.push(onThreshold(thresholdCbkData, data));
thresholdCbkData = [];
}
// To fetch the next page of records, call `fetchNextPage`. // To fetch the next page of records, call `fetchNextPage`.
// If there are more records, `page` will get called again. // If there are more records, `page` will get called again.
// If there are no more records, `done` will get called. // If there are no more records, `done` will get called.
@ -64,11 +59,6 @@ async function readAllData({
console.error(err); console.error(err);
return reject(err); return reject(err);
} }
if (thresholdCbkData.length) {
await Promise.all(insertJobs);
await onThreshold(thresholdCbkData, data);
thresholdCbkData = [];
}
resolve(data); resolve(data);
} }
); );
@ -94,7 +84,7 @@ export async function importData({
api: Api<any>; api: Api<any>;
nocoBaseDataProcessing_v2; nocoBaseDataProcessing_v2;
sDB; sDB;
}): Promise<any> { }): Promise<EntityMap> {
try { try {
// @ts-ignore // @ts-ignore
const records = await readAllData({ const records = await readAllData({
@ -102,26 +92,52 @@ export async function importData({
base, base,
logDetailed, logDetailed,
logBasic, logBasic,
async onThreshold(records, allRecords) { });
const allData = [];
for (let i = 0; i < records.length; i++) { await new Promise(async (resolve) => {
const r = await nocoBaseDataProcessing_v2(sDB, table, records[i]); const readable = records.getStream();
allData.push(r); const allRecordsCount = await records.getCount();
} const promises = [];
let tempData = [];
let importedCount = 0;
readable.on('data', async (record) => {
promises.push(new Promise(async (resolve) => {
const { id: rid, ...fields } = record;
const r = await nocoBaseDataProcessing_v2(sDB, table, { id: rid, fields });
tempData.push(r);
logBasic( if (tempData.length >= BULK_DATA_BATCH_SIZE) {
`:: Importing '${table.title}' data :: ${ readable.pause();
allRecords.length - records.length + 1 let insertArray = tempData.splice(0, tempData.length);
} - ${allRecords.length}` await api.dbTableRow.bulkCreate('nc', projectName, table.id, insertArray);
); logBasic(
await api.dbTableRow.bulkCreate('nc', projectName, table.id, allData); `:: Importing '${table.title}' data :: ${importedCount} - ${Math.min(importedCount + BULK_DATA_BATCH_SIZE, allRecordsCount)}`
}, );
importedCount += insertArray.length;
insertArray = [];
readable.resume();
}
resolve(true);
}));
});
readable.on('end', async () => {
await Promise.all(promises);
if (tempData.length > 0) {
await api.dbTableRow.bulkCreate('nc', projectName, table.id, tempData);
logBasic(
`:: Importing '${table.title}' data :: ${importedCount} - ${Math.min(importedCount + BULK_DATA_BATCH_SIZE, allRecordsCount)}`
);
importedCount += tempData.length;
tempData = [];
}
resolve(true);
});
}); });
return records; return records;
} catch (e) { } catch (e) {
console.log(e); console.log(e);
return 0; return null;
} }
} }
@ -146,7 +162,7 @@ export async function importLTARData({
logBasic: (string) => void; logBasic: (string) => void;
api: Api<any>; api: Api<any>;
insertedAssocRef: { [assocTableId: string]: boolean }; insertedAssocRef: { [assocTableId: string]: boolean };
records?: Array<{ fields: any; id: string }>; records?: EntityMap;
atNcAliasRef: { atNcAliasRef: {
[ncTableId: string]: { [ncTableId: string]: {
[ncTitle: string]: string; [ncTitle: string]: string;
@ -209,49 +225,77 @@ export async function importLTARData({
let nestedLinkCnt = 0; let nestedLinkCnt = 0;
// Iterate over all related M2M associative table // Iterate over all related M2M associative table
for (const assocMeta of assocTableMetas) { for await (const assocMeta of assocTableMetas) {
const assocTableData = []; let assocTableData = [];
let importedCount = 0;
// extract insert data from records // extract insert data from records
for (const record of allData) { await new Promise((resolve) => {
const rec = record.fields; const promises = [];
const readable = allData.getStream();
readable.on('data', async (record) => {
promises.push(new Promise(async (resolve) => {
const { id: _atId, ...rec } = record;
// todo: use actual alias instead of sanitized // todo: use actual alias instead of sanitized
assocTableData.push( assocTableData.push(
...(rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || []).map( ...(rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || []).map(
(id) => ({ (id) => ({
[assocMeta.curCol.title]: record.id, [assocMeta.curCol.title]: record.id,
[assocMeta.refCol.title]: id, [assocMeta.refCol.title]: id,
}) })
) )
); );
}
nestedLinkCnt += assocTableData.length; if (assocTableData.length >= ASSOC_BULK_DATA_BATCH_SIZE) {
// Insert datas as chunks of size `ASSOC_BULK_DATA_BATCH_SIZE` readable.pause();
for ( let insertArray = assocTableData.splice(0, assocTableData.length);
let i = 0; logBasic(
i < assocTableData.length; `:: Importing '${table.title}' LTAR data :: ${importedCount} - ${Math.min(
i += ASSOC_BULK_DATA_BATCH_SIZE importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
) { insertArray.length
logBasic( )}`
`:: Importing '${table.title}' LTAR data :: ${i + 1} - ${Math.min( );
i + ASSOC_BULK_DATA_BATCH_SIZE,
assocTableData.length await api.dbTableRow.bulkCreate(
)}` 'nc',
); projectName,
assocMeta.modelMeta.id,
insertArray
);
console.log( importedCount += insertArray.length;
assocTableData.slice(i, i + ASSOC_BULK_DATA_BATCH_SIZE).length insertArray = [];
); readable.resume();
}
resolve(true);
}));
});
readable.on('end', async () => {
await Promise.all(promises);
if (assocTableData.length >= 0) {
logBasic(
`:: Importing '${table.title}' LTAR data :: ${importedCount} - ${Math.min(
importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
assocTableData.length
)}`
);
await api.dbTableRow.bulkCreate(
'nc',
projectName,
assocMeta.modelMeta.id,
assocTableData
);
await api.dbTableRow.bulkCreate( importedCount += assocTableData.length;
'nc', assocTableData = [];
projectName, }
assocMeta.modelMeta.id, resolve(true);
assocTableData.slice(i, i + ASSOC_BULK_DATA_BATCH_SIZE) });
); });
}
nestedLinkCnt += importedCount;
} }
return nestedLinkCnt; return nestedLinkCnt;
} }

Write Preview
Loading…
Cancel
Save