Browse Source

Merge pull request #6427 from nocodb/fix/job-async

fix: handle async error for streamModelDataAsCsv
pull/6475/head
mertmit 1 year ago committed by GitHub
parent
commit
b2ef45dcc9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 28
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts
  2. 9
      packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

28
packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@ -242,7 +242,11 @@ export class DuplicateProcessor {
let handledLinks = []; let handledLinks = [];
let error = null;
for (const sourceModel of sourceModels) { for (const sourceModel of sourceModels) {
if (error) break;
const dataStream = new Readable({ const dataStream = new Readable({
read() {}, read() {},
}); });
@ -251,12 +255,19 @@ export class DuplicateProcessor {
read() {}, read() {},
}); });
this.exportService.streamModelDataAsCsv({ this.exportService
.streamModelDataAsCsv({
dataStream, dataStream,
linkStream, linkStream,
projectId: sourceProject.id, projectId: sourceProject.id,
modelId: sourceModel.id, modelId: sourceModel.id,
handledMmList: handledLinks, handledMmList: handledLinks,
})
.catch((e) => {
this.logger.error(e);
dataStream.push(null);
linkStream.push(null);
error = e;
}); });
const model = await Model.get(findWithIdentifier(idMap, sourceModel.id)); const model = await Model.get(findWithIdentifier(idMap, sourceModel.id));
@ -284,6 +295,8 @@ export class DuplicateProcessor {
); );
} }
if (error) throw error;
// update external models (has bt to this model) // update external models (has bt to this model)
if (externalModels) { if (externalModels) {
for (const sourceModel of externalModels) { for (const sourceModel of externalModels) {
@ -299,13 +312,22 @@ export class DuplicateProcessor {
read() {}, read() {},
}); });
this.exportService.streamModelDataAsCsv({ let error = null;
this.exportService
.streamModelDataAsCsv({
dataStream, dataStream,
linkStream, linkStream,
projectId: sourceProject.id, projectId: sourceProject.id,
modelId: sourceModel.id, modelId: sourceModel.id,
handledMmList: handledLinks, handledMmList: handledLinks,
_fieldIds: fields, _fieldIds: fields,
})
.catch((e) => {
this.logger.error(e);
dataStream.push(null);
linkStream.push(null);
error = e;
}); });
const headers: string[] = []; const headers: string[] = [];
@ -401,6 +423,8 @@ export class DuplicateProcessor {
}); });
}); });
if (error) throw error;
elapsedTime( elapsedTime(
hrTime, hrTime,
`map existing links to ${model.title}`, `map existing links to ${model.title}`,

9
packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@ -722,15 +722,24 @@ export class ExportService {
dataStream, dataStream,
); );
let error = null;
this.streamModelDataAsCsv({ this.streamModelDataAsCsv({
dataStream, dataStream,
linkStream, linkStream,
projectId: project.id, projectId: project.id,
modelId: model.id, modelId: model.id,
handledMmList, handledMmList,
}).catch((e) => {
this.logger.error(e);
dataStream.push(null);
linkStream.push(null);
error = e;
}); });
await Promise.all([uploadPromise, linkPromise]); await Promise.all([uploadPromise, linkPromise]);
if (error) throw error;
} }
combinedLinkStream.push(null); combinedLinkStream.push(null);

Loading…
Cancel
Save