github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Merge pull request #7439 from nocodb/nc-fix/at-follow-up 

fix: at import follow up
pull/7443/head
Mert E 10 months ago committed by GitHub
parent
0c37f1d4be dc1f1753c3
commit
eb59024c22
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 100 additions and 62 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 29
      packages/nc-gui/components/dlg/AirtableImport.vue
  2. 50
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
  3. 83
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts

29
packages/nc-gui/components/dlg/AirtableImport.vue
Unescape View File

@ -309,6 +309,23 @@ onMounted(async () => {
}
await loadSyncSrc()
})
function downloadLogs(filename: string) {
let text = ''
for (const o of document.querySelectorAll('.nc-modal-airtable-import .log-message')) {
text += `${o.textContent}\n`
}
const element = document.createElement('a')
element.setAttribute('href', `data:text/plain;charset=utf-8,${encodeURIComponent(text)}`)
element.setAttribute('download', filename)
element.style.display = 'none'
document.body.appendChild(element)
element.click()
document.body.removeChild(element)
}
</script>
<template>
@ -447,17 +464,25 @@ onMounted(async () => {
<div class="mb-4 prose-xl font-bold">{{ $t('general.logs') }}</div>
<a-card ref="logRef" :body-style="{ backgroundColor: '#000000', height: '400px', overflow: 'auto' }">
<a-button
v-if="showGoToDashboardButton || goBack"
class="!absolute mr-1 mb-1 z-1 right-0 bottom-0 opacity-40 hover:opacity-100"
size="small"
@click="downloadLogs('at-import-logs.txt')"
>
<component :is="iconMap.download" class="text-green-500" />
</a-button>
<div v-for="({ msg, status }, i) in progress" :key="i">
<div v-if="status === JobStatus.FAILED" class="flex items-center">
<component :is="iconMap.closeCircle" class="text-red-500" />
<span class="text-red-500 ml-2">{{ msg }}</span>
<span class="text-red-500 ml-2 log-message">{{ msg }}</span>
</div>
<div v-else class="flex items-center">
<MdiCurrencyUsd class="text-green-500" />
<span class="text-green-500 ml-2">{{ msg }}</span>
<span class="text-green-500 ml-2 log-message">{{ msg }}</span>
</div>
</div>

50
packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
Unescape View File

@ -221,6 +221,7 @@ export class AtImportProcessor {
rtc.migrationSkipLog.log.push(
`tn[${tbl}] cn[${col}] type[${type}] :: ${reason}`,
);
logWarning(`Skipped ${tbl} :: ${col} (${type}) :: ${reason}`);
};
// mapping table
@ -442,6 +443,11 @@ export class AtImportProcessor {
el.title.toLowerCase() === (value as any).name.toLowerCase(),
)
) {
logWarning(
`Duplicate select option found: ${col.name} :: ${
(value as any).name
}`,
);
continue;
}
options.push({
@ -517,7 +523,7 @@ export class AtImportProcessor {
const col = tblSchema[i].columns[j];
// skip link, lookup, rollup fields in this iteration
if (['foreignKey', 'lookup', 'rollup'].includes(col.type)) {
if (['foreignKey', 'lookup', 'rollup', 'count'].includes(col.type)) {
continue;
}
@ -540,7 +546,7 @@ export class AtImportProcessor {
tblSchema[i].name,
ncName.title,
col.type,
'column type not supported',
'column type not supported yet',
);
continue;
}
@ -1095,7 +1101,7 @@ export class AtImportProcessor {
CONCATENATE: '',
COUNT: 'count',
COUNTA: '',
COUNTALL: '',
COUNTALL: 'count',
MAX: 'max',
MIN: 'min',
OR: '',
@ -1109,7 +1115,7 @@ export class AtImportProcessor {
// Rollup
for (let idx = 0; idx < aTblSchema.length; idx++) {
const aTblColumns = aTblSchema[idx].columns.filter(
(x) => x.type === 'rollup',
(x) => x.type === 'rollup' || x.type === 'count',
);
// parent table ID
@ -1127,16 +1133,19 @@ export class AtImportProcessor {
// fetch associated rollup function
// skip column creation if supported rollup function does not exist
const ncRollupFn = getRollupNcFunction(
aTblColumns[i].typeOptions.formulaTextParsed,
);
const ncRollupFn =
aTblColumns[i].type === 'count'
? 'count'
: getRollupNcFunction(
aTblColumns[i].typeOptions.formulaTextParsed,
);
if (ncRollupFn === '' || ncRollupFn === undefined) {
updateMigrationSkipLog(
srcTableSchema.title,
aTblColumns[i].name,
aTblColumns[i].type,
`rollup function ${aTblColumns[i].typeOptions.formulaTextParsed} not supported`,
`rollup function ${aTblColumns[i].typeOptions.formulaTextParsed} not supported yet`,
);
continue;
}
@ -1161,10 +1170,23 @@ export class AtImportProcessor {
const ncRelationColumnId = await sMap.getNcIdFromAtId(
aTblColumns[i].typeOptions.relationColumnId,
);
const ncRollupColumnId = await sMap.getNcIdFromAtId(
let ncRollupColumnId = await sMap.getNcIdFromAtId(
aTblColumns[i].typeOptions.foreignTableRollupColumnId,
);
if (!ncRollupColumnId && aTblColumns[i].type === 'count') {
const ncRelationColumn = await nc_getColumnSchema(
aTblColumns[i].typeOptions.relationColumnId,
);
const ncRelatedModelId =
ncRelationColumn?.colOptions?.fk_related_model_id;
if (ncRelatedModelId) {
ncRollupColumnId = ncSchema.tablesById[
ncRelatedModelId
].columns.find((x) => x.pk)?.id;
}
}
if (!ncRollupColumnId) {
aTblColumns[i]['srcTableId'] = srcTableId;
nestedRollupTbl.push(aTblColumns[i]);
@ -1185,7 +1207,7 @@ export class AtImportProcessor {
srcTableSchema.title,
aTblColumns[i].name,
aTblColumns[i].type,
'rollup referring to a column type not supported currently',
'rollup referring to a column type not supported yet',
);
continue;
}
@ -1879,7 +1901,7 @@ export class AtImportProcessor {
(x) => x.type === 'lookup',
);
const aTblRollup = aTblSchema[idx].columns.filter(
(x) => x.type === 'rollup',
(x) => x.type === 'rollup' || x.type === 'count',
);
let invalidColumnId = 0;
@ -2033,7 +2055,7 @@ export class AtImportProcessor {
await sMap.getNcNameFromAtId(viewId),
colSchema.title,
colSchema.uidt,
`filter config skipped; filter over date datatype not supported`,
`filter config skipped; filter over date datatype not supported yet`,
);
continue;
}
@ -2172,7 +2194,7 @@ export class AtImportProcessor {
await sMap.getNcNameFromAtId(viewId),
colSchema.title,
colSchema.uidt,
`group config skipped; group over ${datatype} not supported`,
`group config skipped; group over ${datatype} not supported yet`,
);
continue;
}
@ -2329,8 +2351,8 @@ export class AtImportProcessor {
///////////////////////////////////////////////////////////////////////////////
try {
logBasic('SDK initialized');
logDetailed('Base initialization started');
// delete base if already exists
logDetailed('Base initialized');

83
packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts
Unescape View File

@ -10,6 +10,7 @@ import type { TableType } from 'nocodb-sdk';
const BULK_DATA_BATCH_COUNT = 20; // check size for every 100 records
const BULK_DATA_BATCH_SIZE = 50 * 1024; // in bytes
const BULK_LINK_BATCH_COUNT = 1000; // process 1000 records at a time
const BULK_PARALLEL_PROCESS = 5;
interface AirtableImportContext {
@ -307,12 +308,10 @@ export async function importLTARData({
}
let nestedLinkCnt = 0;
let importedCount = 0;
let assocTableData = [];
// Iterate over all related M2M associative table
for await (const assocMeta of assocTableMetas) {
let assocTableData = [];
let importedCount = 0;
let tempCount = 0;
for (const assocMeta of assocTableMetas) {
// extract link data from records
await new Promise((resolve, reject) => {
const promises = [];
@ -333,40 +332,35 @@ export async function importLTARData({
[assocMeta.refCol.title]: id,
})),
);
tempCount++;
if (tempCount >= BULK_DATA_BATCH_COUNT) {
if (sizeof(assocTableData) >= BULK_DATA_BATCH_SIZE) {
readable.pause();
let insertArray = assocTableData.splice(
0,
assocTableData.length,
);
logBasic(
`:: Importing '${
table.title
}' LTAR data :: ${importedCount} - ${Math.min(
importedCount + insertArray.length,
insertArray.length,
)}`,
);
await services.bulkDataService.bulkDataInsert({
baseName,
tableName: assocMeta.modelMeta.id,
body: insertArray,
cookie: {},
skip_hooks: true,
});
importedCount += insertArray.length;
insertArray = [];
readable.resume();
}
tempCount = 0;
if (assocTableData.length >= BULK_LINK_BATCH_COUNT) {
readable.pause();
let insertArray = assocTableData.splice(
0,
assocTableData.length,
);
logBasic(
`:: Importing '${
table.title
}' LTAR data :: ${importedCount} - ${
importedCount + insertArray.length
}`,
);
await services.bulkDataService.bulkDataInsert({
baseName,
tableName: assocMeta.modelMeta.id,
body: insertArray,
cookie: {},
skip_hooks: true,
});
importedCount += insertArray.length;
insertArray = [];
readable.resume();
}
resolve(true);
} catch (e) {
@ -383,12 +377,9 @@ export async function importLTARData({
// insert remaining data
if (assocTableData.length >= 0) {
logBasic(
`:: Importing '${
table.title
}' LTAR data :: ${importedCount} - ${Math.min(
importedCount + assocTableData.length,
assocTableData.length,
)}`,
`:: Importing '${table.title}' LTAR data :: ${importedCount} - ${
importedCount + assocTableData.length
}`,
);
await services.bulkDataService.bulkDataInsert({
@ -403,14 +394,14 @@ export async function importLTARData({
assocTableData = [];
}
nestedLinkCnt += importedCount;
resolve(true);
} catch (e) {
reject(e);
}
});
});
nestedLinkCnt += importedCount;
}
return nestedLinkCnt;
}

Write Preview
Loading…
Cancel
Save