github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: better logging for jobs 

Signed-off-by: mertmit <mertmit99@gmail.com>
pull/5621/head
mertmit 1 year ago
parent
9fab04b265
commit
e68d2d376c
4 changed files with 120 additions and 84 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 73
      packages/nocodb/src/modules/jobs/export-import/duplicate.processor.ts
  2. 27
      packages/nocodb/src/modules/jobs/export-import/export.service.ts
  3. 81
      packages/nocodb/src/modules/jobs/export-import/import.service.ts
  4. 23
      packages/nocodb/src/modules/jobs/helpers.ts

73
packages/nocodb/src/modules/jobs/export-import/duplicate.processor.ts
Unescape View File

@ -3,41 +3,22 @@ import { Process, Processor } from '@nestjs/bull';
import { Job } from 'bull'; import { Job } from 'bull';
import papaparse from 'papaparse'; import papaparse from 'papaparse';
import { UITypes } from 'nocodb-sdk'; import { UITypes } from 'nocodb-sdk';
import { Logger } from '@nestjs/common';
import { Base, Column, Model, Project } from '../../../models'; import { Base, Column, Model, Project } from '../../../models';
import { ProjectsService } from '../../../services/projects.service'; import { ProjectsService } from '../../../services/projects.service';
import { findWithIdentifier } from '../../../helpers/exportImportHelpers'; import { findWithIdentifier } from '../../../helpers/exportImportHelpers';
import { BulkDataAliasService } from '../../../services/bulk-data-alias.service'; import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs'; import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs';
import { elapsedTime, initTime } from '../helpers';
import { ExportService } from './export.service'; import { ExportService } from './export.service';
import { ImportService } from './import.service'; import { ImportService } from './import.service';
import type { LinkToAnotherRecordColumn } from '../../../models';
const DEBUG = false;
const debugLog = function (...args: any[]) {
if (DEBUG) {
console.log(...args);
}
};
const initTime = function () {
return {
hrTime: process.hrtime(),
};
};
const elapsedTime = function (
time: { hrTime: [number, number] },
label?: string,
) {
const elapsedS = process.hrtime(time.hrTime)[0].toFixed(3);
const elapsedMs = process.hrtime(time.hrTime)[1] / 1000000;
if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
time.hrTime = process.hrtime();
};
@Processor(JOBS_QUEUE) @Processor(JOBS_QUEUE)
export class DuplicateProcessor { export class DuplicateProcessor {
private readonly logger = new Logger(
`${JOBS_QUEUE}:${DuplicateProcessor.name}`,
);
constructor( constructor(
private readonly exportService: ExportService, private readonly exportService: ExportService,
private readonly importService: ImportService, private readonly importService: ImportService,
@ -77,7 +58,11 @@ export class DuplicateProcessor {
excludeHooks, excludeHooks,
}); });
elapsedTime(hrTime, 'serializeModels'); elapsedTime(
hrTime,
`serialize models schema for ${base.project_id}::${base.id}`,
'duplicateBase',
);
if (!exportedModels) { if (!exportedModels) {
throw new Error(`Export failed for base '${base.id}'`); throw new Error(`Export failed for base '${base.id}'`);
@ -95,7 +80,7 @@ export class DuplicateProcessor {
req: req, req: req,
}); });
elapsedTime(hrTime, 'importModels'); elapsedTime(hrTime, `import models schema`, 'duplicateBase');
if (!idMap) { if (!idMap) {
throw new Error(`Import failed for base '${base.id}'`); throw new Error(`Import failed for base '${base.id}'`);
@ -166,7 +151,11 @@ export class DuplicateProcessor {
}) })
)[0]; )[0];
elapsedTime(hrTime, 'serializeModel'); elapsedTime(
hrTime,
`serialize model schema for ${modelId}`,
'duplicateModel',
);
if (!exportedModel) { if (!exportedModel) {
throw new Error(`Export failed for base '${base.id}'`); throw new Error(`Export failed for base '${base.id}'`);
@ -184,7 +173,7 @@ export class DuplicateProcessor {
externalModels: relatedModels, externalModels: relatedModels,
}); });
elapsedTime(hrTime, 'reimportModelSchema'); elapsedTime(hrTime, 'import model schema', 'duplicateModel');
if (!idMap) { if (!idMap) {
throw new Error(`Import failed for model '${modelId}'`); throw new Error(`Import failed for model '${modelId}'`);
@ -220,7 +209,7 @@ export class DuplicateProcessor {
externalModels: relatedModels, externalModels: relatedModels,
}); });
elapsedTime(hrTime, 'reimportModelData'); elapsedTime(hrTime, 'import model data', 'duplicateModel');
} }
return await Model.get(findWithIdentifier(idMap, sourceModel.id)); return await Model.get(findWithIdentifier(idMap, sourceModel.id));
@ -274,7 +263,6 @@ export class DuplicateProcessor {
destProject, destProject,
destBase, destBase,
destModel: model, destModel: model,
debugLog,
}); });
handledLinks = await this.importService.importLinkFromCsvStream({ handledLinks = await this.importService.importLinkFromCsvStream({
@ -283,10 +271,13 @@ export class DuplicateProcessor {
destProject, destProject,
destBase, destBase,
handledLinks, handledLinks,
debugLog,
}); });
elapsedTime(hrTime, model.title); elapsedTime(
hrTime,
`import data and links for ${model.title}`,
'importModelsData',
);
} }
// update external models (has bt to this model) // update external models (has bt to this model)
@ -341,18 +332,18 @@ export class DuplicateProcessor {
headers.push(childCol.column_name); headers.push(childCol.column_name);
} else { } else {
headers.push(null); headers.push(null);
debugLog('child column not found', id); this.logger.error(`child column not found (${id})`);
} }
} else { } else {
headers.push(col.column_name); headers.push(col.column_name);
} }
} else { } else {
headers.push(null); headers.push(null);
debugLog('column not found', id); this.logger.error(`column not found (${id})`);
} }
} else { } else {
headers.push(null); headers.push(null);
debugLog('header not found', header); this.logger.error(`id not found (${header})`);
} }
} }
parser.resume(); parser.resume();
@ -378,7 +369,7 @@ export class DuplicateProcessor {
raw: true, raw: true,
}); });
} catch (e) { } catch (e) {
console.log(e); this.logger.error(e);
} }
chunk = []; chunk = [];
parser.resume(); parser.resume();
@ -397,7 +388,7 @@ export class DuplicateProcessor {
raw: true, raw: true,
}); });
} catch (e) { } catch (e) {
console.log(e); this.logger.error(e);
} }
chunk = []; chunk = [];
} }
@ -406,7 +397,11 @@ export class DuplicateProcessor {
}); });
}); });
elapsedTime(hrTime, `external bt ${model.title}`); elapsedTime(
hrTime,
`map existing links to ${model.title}`,
'importModelsData',
);
} }
} }
} }

27
packages/nocodb/src/modules/jobs/export-import/export.service.ts
Unescape View File

@ -1,7 +1,7 @@
import { Readable } from 'stream'; import { Readable } from 'stream';
import { UITypes, ViewTypes } from 'nocodb-sdk'; import { UITypes, ViewTypes } from 'nocodb-sdk';
import { unparse } from 'papaparse'; import { unparse } from 'papaparse';
import { Injectable } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
import { getViewAndModelByAliasOrId } from '../../../modules/datas/helpers'; import { getViewAndModelByAliasOrId } from '../../../modules/datas/helpers';
import { import {
@ -12,11 +12,14 @@ import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2';
import { NcError } from '../../../helpers/catchError'; import { NcError } from '../../../helpers/catchError';
import { Base, Hook, Model, Project } from '../../../models'; import { Base, Hook, Model, Project } from '../../../models';
import { DatasService } from '../../../services/datas.service'; import { DatasService } from '../../../services/datas.service';
import { elapsedTime, initTime } from '../helpers';
import type { BaseModelSqlv2 } from '../../../db/BaseModelSqlv2'; import type { BaseModelSqlv2 } from '../../../db/BaseModelSqlv2';
import type { LinkToAnotherRecordColumn, View } from '../../../models'; import type { View } from '../../../models';
@Injectable() @Injectable()
export class ExportService { export class ExportService {
private readonly logger = new Logger(ExportService.name);
constructor(private datasService: DatasService) {} constructor(private datasService: DatasService) {}
async serializeModels(param: { async serializeModels(param: {
@ -422,7 +425,7 @@ export class ExportService {
true, true,
); );
} catch (e) { } catch (e) {
console.error(e); this.logger.error(e);
throw e; throw e;
} }
@ -488,7 +491,7 @@ export class ExportService {
true, true,
); );
} catch (e) { } catch (e) {
console.error(e); this.logger.error(e);
throw e; throw e;
} }
@ -597,6 +600,8 @@ export class ExportService {
} }
async exportBase(param: { path: string; baseId: string }) { async exportBase(param: { path: string; baseId: string }) {
const hrTime = initTime();
const base = await Base.get(param.baseId); const base = await Base.get(param.baseId);
if (!base) if (!base)
@ -613,6 +618,12 @@ export class ExportService {
modelIds: models.map((m) => m.id), modelIds: models.map((m) => m.id),
}); });
elapsedTime(
hrTime,
`serialize models for ${base.project_id}::${base.id}`,
'exportBase',
);
const exportData = { const exportData = {
id: `${project.id}::${base.id}`, id: `${project.id}::${base.id}`,
models: exportedModels, models: exportedModels,
@ -669,7 +680,7 @@ export class ExportService {
}); });
linkStream.on('error', (e) => { linkStream.on('error', (e) => {
console.error(e); this.logger.error(e);
resolve(null); resolve(null);
}); });
}); });
@ -693,6 +704,12 @@ export class ExportService {
combinedLinkStream.push(null); combinedLinkStream.push(null);
await uploadLinkPromise; await uploadLinkPromise;
elapsedTime(
hrTime,
`export base ${base.project_id}::${base.id}`,
'exportBase',
);
} catch (e) { } catch (e) {
throw NcError.badRequest(e); throw NcError.badRequest(e);
} }

81
packages/nocodb/src/modules/jobs/export-import/import.service.ts
Unescape View File

@ -1,5 +1,5 @@
import { UITypes, ViewTypes } from 'nocodb-sdk'; import { UITypes, ViewTypes } from 'nocodb-sdk';
import { Injectable } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import papaparse from 'papaparse'; import papaparse from 'papaparse';
import { import {
findWithIdentifier, findWithIdentifier,
@ -27,12 +27,15 @@ import { HooksService } from '../../../services/hooks.service';
import { ViewsService } from '../../../services/views.service'; import { ViewsService } from '../../../services/views.service';
import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2'; import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2';
import { BulkDataAliasService } from '../../../services/bulk-data-alias.service'; import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
import { elapsedTime, initTime } from '../helpers';
import type { Readable } from 'stream'; import type { Readable } from 'stream';
import type { ViewCreateReqType } from 'nocodb-sdk'; import type { ViewCreateReqType } from 'nocodb-sdk';
import type { LinkToAnotherRecordColumn, User, View } from '../../../models'; import type { LinkToAnotherRecordColumn, User, View } from '../../../models';
@Injectable() @Injectable()
export class ImportService { export class ImportService {
private readonly logger = new Logger(ImportService.name);
constructor( constructor(
private tablesService: TablesService, private tablesService: TablesService,
private columnsService: ColumnsService, private columnsService: ColumnsService,
@ -60,6 +63,8 @@ export class ImportService {
req: any; req: any;
externalModels?: Model[]; externalModels?: Model[];
}) { }) {
const hrTime = initTime();
// structured id to db id // structured id to db id
const idMap = new Map<string, string>(); const idMap = new Map<string, string>();
const externalIdMap = new Map<string, string>(); const externalIdMap = new Map<string, string>();
@ -112,6 +117,8 @@ export class ImportService {
} }
} }
elapsedTime(hrTime, 'generate id map for external models', 'importModels');
// create tables with static columns // create tables with static columns
for (const data of param.data) { for (const data of param.data) {
const modelData = data.model; const modelData = data.model;
@ -149,9 +156,11 @@ export class ImportService {
tableReferences.set(modelData.id, table); tableReferences.set(modelData.id, table);
} }
elapsedTime(hrTime, 'create tables with static columns', 'importModels');
const referencedColumnSet = []; const referencedColumnSet = [];
// create columns with reference to other columns // create LTAR columns
for (const data of param.data) { for (const data of param.data) {
const modelData = data.model; const modelData = data.model;
const table = tableReferences.get(modelData.id); const table = tableReferences.get(modelData.id);
@ -160,7 +169,6 @@ export class ImportService {
(a) => a.uidt === UITypes.LinkToAnotherRecord, (a) => a.uidt === UITypes.LinkToAnotherRecord,
); );
// create columns with reference to other columns
for (const col of linkedColumnSet) { for (const col of linkedColumnSet) {
if (col.colOptions) { if (col.colOptions) {
const colOptions = col.colOptions; const colOptions = col.colOptions;
@ -702,6 +710,8 @@ export class ImportService {
); );
} }
elapsedTime(hrTime, 'create LTAR columns', 'importModels');
const sortedReferencedColumnSet = []; const sortedReferencedColumnSet = [];
// sort referenced columns to avoid referencing before creation // sort referenced columns to avoid referencing before creation
@ -815,6 +825,8 @@ export class ImportService {
} }
} }
elapsedTime(hrTime, 'create referenced columns', 'importModels');
// create views // create views
for (const data of param.data) { for (const data of param.data) {
const modelData = data.model; const modelData = data.model;
@ -931,6 +943,8 @@ export class ImportService {
} }
} }
elapsedTime(hrTime, 'create views', 'importModels');
// create hooks // create hooks
for (const data of param.data) { for (const data of param.data) {
if (!data?.hooks) break; if (!data?.hooks) break;
@ -973,6 +987,8 @@ export class ImportService {
} }
} }
elapsedTime(hrTime, 'create hooks', 'importModels');
return idMap; return idMap;
} }
@ -1115,16 +1131,10 @@ export class ImportService {
file?: any; file?: any;
}; };
req: any; req: any;
debug?: boolean;
}) { }) {
const { user, projectId, baseId, src, req } = param; const hrTime = initTime();
const debug = param?.debug === true;
const debugLog = (...args: any[]) => { const { user, projectId, baseId, src, req } = param;
if (!debug) return;
console.log(...args);
};
const destProject = await Project.get(projectId); const destProject = await Project.get(projectId);
const destBase = await Base.get(baseId); const destBase = await Base.get(baseId);
@ -1133,15 +1143,6 @@ export class ImportService {
throw NcError.badRequest('Project or Base not found'); throw NcError.badRequest('Project or Base not found');
} }
let start = process.hrtime();
const elapsedTime = function (label?: string) {
const elapsedS = process.hrtime(start)[0].toFixed(3);
const elapsedMs = process.hrtime(start)[1] / 1000000;
if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
start = process.hrtime();
};
switch (src.type) { switch (src.type) {
case 'local': { case 'local': {
const path = src.path.replace(/\/$/, ''); const path = src.path.replace(/\/$/, '');
@ -1153,7 +1154,7 @@ export class ImportService {
await storageAdapter.fileRead(`${path}/schema.json`), await storageAdapter.fileRead(`${path}/schema.json`),
); );
elapsedTime('read schema'); elapsedTime(hrTime, 'read schema from file', 'importBase');
// store fk_mm_model_id (mm) to link once // store fk_mm_model_id (mm) to link once
let handledLinks = []; let handledLinks = [];
@ -1166,7 +1167,7 @@ export class ImportService {
req, req,
}); });
elapsedTime('import models'); elapsedTime(hrTime, 'import models schema', 'importBase');
if (idMap) { if (idMap) {
const files = await (storageAdapter as any).getDirectoryList( const files = await (storageAdapter as any).getDirectoryList(
@ -1189,7 +1190,7 @@ export class ImportService {
const model = await Model.get(modelId); const model = await Model.get(modelId);
debugLog(`Importing ${model.title}...`); this.logger.debug(`Importing ${model.title}...`);
await this.importDataFromCsvStream({ await this.importDataFromCsvStream({
idMap, idMap,
@ -1197,14 +1198,17 @@ export class ImportService {
destProject, destProject,
destBase, destBase,
destModel: model, destModel: model,
debugLog,
}); });
elapsedTime(`import ${model.title}`); elapsedTime(
hrTime,
`import data for ${model.title}`,
'importBase',
);
} }
// reset timer // reset timer
elapsedTime(); elapsedTime(hrTime);
const linkReadStream = await ( const linkReadStream = await (
storageAdapter as any storageAdapter as any
@ -1216,8 +1220,9 @@ export class ImportService {
destProject, destProject,
destBase, destBase,
handledLinks, handledLinks,
debugLog,
}); });
elapsedTime(hrTime, `import links`, 'importBase');
} }
} catch (e) { } catch (e) {
throw new Error(e); throw new Error(e);
@ -1237,12 +1242,9 @@ export class ImportService {
destProject: Project; destProject: Project;
destBase: Base; destBase: Base;
destModel: Model; destModel: Model;
debugLog?: (...args: any[]) => void;
}): Promise<void> { }): Promise<void> {
const { idMap, dataStream, destBase, destProject, destModel } = param; const { idMap, dataStream, destBase, destProject, destModel } = param;
const debugLog = param.debugLog || (() => {});
const headers: string[] = []; const headers: string[] = [];
let chunk = []; let chunk = [];
@ -1269,18 +1271,20 @@ export class ImportService {
headers.push(childCol.column_name); headers.push(childCol.column_name);
} else { } else {
headers.push(null); headers.push(null);
debugLog('child column not found', header); this.logger.error(
`child column not found (${col.colOptions.fk_child_column_id})`,
);
} }
} else { } else {
headers.push(col.column_name); headers.push(col.column_name);
} }
} else { } else {
headers.push(null); headers.push(null);
debugLog('column not found', header); this.logger.error(`column not found (${id})`);
} }
} else { } else {
headers.push(null); headers.push(null);
debugLog('header not found', header); this.logger.error(`id not found (${header})`);
} }
} }
parser.resume(); parser.resume();
@ -1308,7 +1312,7 @@ export class ImportService {
raw: true, raw: true,
}); });
} catch (e) { } catch (e) {
console.log(e); this.logger.error(e);
} }
chunk = []; chunk = [];
parser.resume(); parser.resume();
@ -1329,7 +1333,7 @@ export class ImportService {
raw: true, raw: true,
}); });
} catch (e) { } catch (e) {
console.log(e); this.logger.error(e);
} }
chunk = []; chunk = [];
} }
@ -1346,12 +1350,9 @@ export class ImportService {
destProject: Project; destProject: Project;
destBase: Base; destBase: Base;
handledLinks: string[]; handledLinks: string[];
debugLog?: (...args: any[]) => void;
}): Promise<string[]> { }): Promise<string[]> {
const { idMap, linkStream, destBase, destProject, handledLinks } = param; const { idMap, linkStream, destBase, destProject, handledLinks } = param;
const debugLog = param.debugLog || (() => {});
const lChunks: Record<string, any[]> = {}; // fk_mm_model_id: { rowId, childId }[] const lChunks: Record<string, any[]> = {}; // fk_mm_model_id: { rowId, childId }[]
const insertChunks = async () => { const insertChunks = async () => {
@ -1369,7 +1370,7 @@ export class ImportService {
}); });
lChunks[k] = []; lChunks[k] = [];
} catch (e) { } catch (e) {
console.log(e); this.logger.error(e);
} }
} }
}; };
@ -1452,7 +1453,7 @@ export class ImportService {
[mm.child]: child, [mm.child]: child,
}); });
} else { } else {
debugLog('column not found', columnId); this.logger.error(`column not found (${columnId})`);
} }
parser.resume(); parser.resume();

23
packages/nocodb/src/modules/jobs/helpers.ts
Unescape View File

@ -0,0 +1,23 @@
import { Logger } from '@nestjs/common';
import { JOBS_QUEUE } from '../../interface/Jobs';
export const initTime = function () {
return {
hrTime: process.hrtime(),
};
};
export const elapsedTime = function (
time: { hrTime: [number, number] },
label?: string,
context?: string,
) {
const elapsedS = process.hrtime(time.hrTime)[0].toFixed(3);
const elapsedMs = process.hrtime(time.hrTime)[1] / 1000000;
if (label)
Logger.debug(
`${label}: ${elapsedS}s ${elapsedMs}ms`,
`${JOBS_QUEUE}${context ? `:${context}` : ''}`,
);
time.hrTime = process.hrtime();
};
Write Preview
Loading…
Cancel
Save