Browse Source

fix: LTAR data insertion

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/2218/head
Pranav C 2 years ago
parent
commit
603c5a24de
  1. 2
      packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts
  2. 23
      packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts

2
packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts

@ -1409,7 +1409,7 @@ export default async (
console.log(e); console.log(e);
}); });
tempArr.push(...rs); tempArr.push(...(rs || []));
} }
rec[key] = JSON.stringify(tempArr); rec[key] = JSON.stringify(tempArr);
} }

23
packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts

@ -2,7 +2,7 @@ import { AirtableBase } from 'airtable/lib/airtable_base';
import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk'; import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk';
const BULK_DATA_BATCH_SIZE = 2000; const BULK_DATA_BATCH_SIZE = 2000;
const ASSOC_BULK_DATA_BATCH_SIZE = 2000; const ASSOC_BULK_DATA_BATCH_SIZE = 5000;
async function readAllData({ async function readAllData({
table, table,
@ -199,14 +199,14 @@ export async function importLTARData({
let nestedLinkCnt = 0; let nestedLinkCnt = 0;
// Iterate over all related M2M associative table // Iterate over all related M2M associative table
for (const assocMeta of assocTableMetas) { for (const assocMeta of assocTableMetas) {
const insertData = []; const assocTableData = [];
// extract insert data from records // extract insert data from records
for (const record of allData) { for (const record of allData) {
const rec = record.fields; const rec = record.fields;
// todo: use actual alias instead of sanitized // todo: use actual alias instead of sanitized
insertData.push( assocTableData.push(
...(rec?.[assocMeta.colMeta.title] || []).map(id => ({ ...(rec?.[assocMeta.colMeta.title] || []).map(id => ({
[assocMeta.curCol.title]: record.id, [assocMeta.curCol.title]: record.id,
[assocMeta.refCol.title]: id [assocMeta.refCol.title]: id
@ -214,20 +214,29 @@ export async function importLTARData({
); );
} }
nestedLinkCnt += insertData.length; nestedLinkCnt += assocTableData.length;
// Insert datas as chunks of size `ASSOC_BULK_DATA_BATCH_SIZE` // Insert datas as chunks of size `ASSOC_BULK_DATA_BATCH_SIZE`
for (let i = 0; i < insertData.length; i += ASSOC_BULK_DATA_BATCH_SIZE) { for (
let i = 0;
i < assocTableData.length;
i += ASSOC_BULK_DATA_BATCH_SIZE
) {
logBasic( logBasic(
`:: Importing '${table.title}' LTAR data :: ${i + 1} - ${Math.min( `:: Importing '${table.title}' LTAR data :: ${i + 1} - ${Math.min(
i + ASSOC_BULK_DATA_BATCH_SIZE, i + ASSOC_BULK_DATA_BATCH_SIZE,
insertData.length assocTableData.length
)}` )}`
); );
console.log(
assocTableData.slice(i, i + ASSOC_BULK_DATA_BATCH_SIZE).length
);
await api.dbTableRow.bulkCreate( await api.dbTableRow.bulkCreate(
'nc', 'nc',
projectName, projectName,
assocMeta.modelMeta.id, assocMeta.modelMeta.id,
insertData.slice(i, ASSOC_BULK_DATA_BATCH_SIZE) assocTableData.slice(i, i + ASSOC_BULK_DATA_BATCH_SIZE)
); );
} }
} }

Loading…
Cancel
Save