github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Merge pull request #7391 from nocodb/nc-fix/dg-at-import 

fix: various at import issues
pull/7394/head
Mert E 10 months ago committed by GitHub
parent
cf138af5e5 8c798de887
commit
9d52187d1b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 85 additions and 59 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 121
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
  2. 12
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
  3. 11
      packages/nocodb/src/services/tables.service.ts

121
packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
Unescape View File

@ -146,6 +146,11 @@ export class AtImportProcessor {
this.debugLog(log);
};
const logWarning = (log) => {
this.jobsLogService.sendLog(job, { message: `WARNING: ${log}` });
this.debugLog(log);
};
const logDetailed = (log) => {
if (debugMode) this.jobsLogService.sendLog(job, { message: log });
this.debugLog(log);
@ -291,7 +296,7 @@ export class AtImportProcessor {
rating: UITypes.Rating,
formula: UITypes.Formula,
rollup: UITypes.Rollup,
count: UITypes.Count,
count: UITypes.Rollup,
lookup: UITypes.Lookup,
autoNumber: UITypes.AutoNumber,
barcode: UITypes.SingleLineText,
@ -430,19 +435,14 @@ export class AtImportProcessor {
if ((value as any).name === '') {
(value as any).name = 'nc_empty';
}
// enumerate duplicates (we don't allow them)
// TODO fix record mapping (this causes every record to map first option,
// we can't handle them using data api as they don't provide option id
// within data we might instead get the correct mapping from schema file )
let dupNo = 1;
const defaultName = (value as any).name;
while (
// skip duplicates (we don't allow them)
if (
options.find(
(el) =>
el.title.toLowerCase() === (value as any).name.toLowerCase(),
)
) {
(value as any).name = `${defaultName}_${dupNo++}`;
continue;
}
options.push({
order: order++,
@ -865,15 +865,6 @@ export class AtImportProcessor {
);
}
// check if already a column exists with this name?
const duplicate = childTblSchema.columns.find(
(x) => x.title === aTblLinkColumns[i].name,
);
const suffix = duplicate ? '_2' : '';
if (duplicate)
if (enableErrorLogs)
console.log(`## Duplicate ${aTblLinkColumns[i].name}`);
// rename
// note that: current rename API requires us to send all parameters,
// not just title being renamed
@ -900,12 +891,12 @@ export class AtImportProcessor {
updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(
(x) => x.title === aTblLinkColumns[i].name + suffix,
(x) => x.title === ncName.title,
)?.id;
await sMap.addToMappingTbl(
aTblLinkColumns[i].id,
ncId,
aTblLinkColumns[i].name + suffix,
ncName.title,
ncTbl.id,
);
}
@ -1100,7 +1091,7 @@ export class AtImportProcessor {
ARRAYCOMPACT: '',
ARRAYJOIN: '',
ARRAYUNIQUE: '',
AVERAGE: 'average',
AVERAGE: 'avg',
CONCATENATE: '',
COUNT: 'count',
COUNTA: '',
@ -1206,35 +1197,41 @@ export class AtImportProcessor {
logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`);
const _perfStart = recordPerfStart();
const ncTbl: any = await this.columnsService.columnAdd({
tableId: srcTableId,
column: {
uidt: UITypes.Rollup,
title: ncName.title,
column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId,
fk_rollup_column_id: ncRollupColumnId,
rollup_function: ncRollupFn,
},
req: {
user: syncDB.user.email,
clientIp: '',
},
user: syncDB.user,
});
recordPerfStats(_perfStart, 'dbTableColumn.create');
try {
const ncTbl: any = await this.columnsService.columnAdd({
tableId: srcTableId,
column: {
uidt: UITypes.Rollup,
title: ncName.title,
column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId,
fk_rollup_column_id: ncRollupColumnId,
rollup_function: ncRollupFn,
},
req: {
user: syncDB.user.email,
clientIp: '',
},
user: syncDB.user,
});
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl);
updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(
(x) => x.title === aTblColumns[i].name,
)?.id;
await sMap.addToMappingTbl(
aTblColumns[i].id,
ncId,
aTblColumns[i].name,
ncTbl.id,
);
const ncId = ncTbl.columns.find(
(x) => x.title === aTblColumns[i].name,
)?.id;
await sMap.addToMappingTbl(
aTblColumns[i].id,
ncId,
aTblColumns[i].name,
ncTbl.id,
);
} catch (e) {
logWarning(
`Skipped creating rollup column ${aTblColumns[i].name} :: ${e.message}`,
);
}
}
}
}
@ -1812,9 +1809,14 @@ export class AtImportProcessor {
},
req: { user: syncDB.user, clientIp: '' },
})
.catch((e) =>
e.message ? logBasic(`NOTICE: ${e.message}`) : console.log(e),
),
.catch((e) => {
if (e.message) {
// TODO enable after fixing user invite role issue
// logWarning(e.message);
} else {
console.log(e);
}
}),
);
recordPerfStats(_perfStart, 'auth.baseUserAdd');
}
@ -2106,12 +2108,16 @@ export class AtImportProcessor {
// insert filters
for (let i = 0; i < ncFilters.length; i++) {
const _perfStart = recordPerfStart();
await this.filtersService.filterCreate({
viewId: viewId,
filter: ncFilters[i],
user: syncDB.user,
req: {},
});
try {
await this.filtersService.filterCreate({
viewId: viewId,
filter: ncFilters[i],
user: syncDB.user,
req: {},
});
} catch (e) {
logWarning(`Skipped creating filter for ${viewId} :: ${e.message}`);
}
recordPerfStats(_perfStart, 'dbTableFilter.create');
rtc.filter++;
@ -2498,6 +2504,7 @@ export class AtImportProcessor {
await this.tablesService.tableDelete({
tableId: table.id,
user: syncDB.user,
forceDeleteRelations: true,
});
}
if (e.message) {

12
packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
Unescape View File

@ -50,12 +50,24 @@ async function initialize(shareId, appId?: string) {
};
});
const headers = hreq.match(/(?<=var headers =)(.*)(?=;)/g);
const link = hreq.match(/(?<=fetch\(")(\\.*)(?=")/g);
if (!headers || !link) {
throw {
message:
'Please ensure www.airtable.com/<SharedBaseID> is available for public access. Refer https://bit.ly/3x0OdXI for details',
};
}
info.headers = JSON.parse(
hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
);
info.link = unicodeToChar(
hreq.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(),
);
info.baseInfo = decodeURIComponent(info.link)
.match(/{(.*)}/g)[0]
.split('&')

11
packages/nocodb/src/services/tables.service.ts
Unescape View File

@ -162,7 +162,12 @@ export class TablesService {
return Model.updateOrder(param.tableId, param.order);
}
async tableDelete(param: { tableId: string; user: User; req?: any }) {
async tableDelete(param: {
tableId: string;
user: User;
forceDeleteRelations?: boolean;
req?: any;
}) {
const table = await Model.getByIdOrName({ id: param.tableId });
await table.getColumns();
@ -171,7 +176,9 @@ export class TablesService {
const relationColumns = table.columns.filter((c) => isLinksOrLTAR(c));
if (relationColumns?.length && !source.isMeta()) {
const deleteRelations = source.isMeta() || param.forceDeleteRelations;
if (relationColumns?.length && !deleteRelations) {
const referredTables = await Promise.all(
relationColumns.map(async (c) =>
c

Write Preview
Loading…
Cancel
Save