github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: debugLog for import/export 

Signed-off-by: mertmit <mertmit99@gmail.com>
feat/export-nest-dir-restructure
mertmit 2 years ago committed by starbirdtech383
parent
8d3bb88745
commit
c0c03f4a83
1 changed files with 53 additions and 32 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 85
      packages/nocodb/src/services/exportImport/import.svc.ts

85
packages/nocodb/src/services/exportImport/import.svc.ts
Unescape View File

@ -573,13 +573,20 @@ export async function importBase(param: {
}) { }) {
const { user, projectId, baseId, src, req } = param; const { user, projectId, baseId, src, req } = param;
const debug = req.params.debug === 'true';
const debugLog = (...args: any[]) => {
if (!debug) return;
console.log(...args);
}
let start = process.hrtime(); let start = process.hrtime();
let elapsed_time = function(label: string){ let elapsedTime = function(label?: string){
const elapsedS = (process.hrtime(start)[0]).toFixed(3); const elapsedS = (process.hrtime(start)[0]).toFixed(3);
const elapsedMs = process.hrtime(start)[1] / 1000000; const elapsedMs = process.hrtime(start)[1] / 1000000;
console.log(`${label}: ${elapsedS}s ${elapsedMs}ms`); if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
start = process.hrtime(); start = process.hrtime();
} }
switch (src.type) { switch (src.type) {
@ -591,7 +598,7 @@ export async function importBase(param: {
try { try {
const schema = JSON.parse(await storageAdapter.fileRead(`${path}/schema.json`)); const schema = JSON.parse(await storageAdapter.fileRead(`${path}/schema.json`));
elapsed_time('read schema'); elapsedTime('read schema');
// store fk_mm_model_id (mm) to link once // store fk_mm_model_id (mm) to link once
const handledLinks = []; const handledLinks = [];
@ -604,7 +611,7 @@ export async function importBase(param: {
req, req,
}); });
elapsed_time('import models'); elapsedTime('import models');
if (idMap) { if (idMap) {
const files = await storageAdapter.getDirectoryList(`${path}/data`); const files = await storageAdapter.getDirectoryList(`${path}/data`);
@ -626,7 +633,7 @@ export async function importBase(param: {
const model = await Model.get(modelId); const model = await Model.get(modelId);
console.log(`Importing ${model.title}...`); debugLog(`Importing ${model.title}...`);
await new Promise(async (resolve) => { await new Promise(async (resolve) => {
papaparse.parse(readStream, { papaparse.parse(readStream, {
@ -650,9 +657,8 @@ export async function importBase(param: {
} else { } else {
headers.push(col.title); headers.push(col.title);
} }
} else { } else {
console.log(header); debugLog(header);
} }
} }
parser.resume(); parser.resume();
@ -667,17 +673,22 @@ export async function importBase(param: {
chunk.push(row); chunk.push(row);
if (chunk.length > 1000) { if (chunk.length > 1000) {
parser.pause(); parser.pause();
elapsed_time('before chunk'); elapsedTime('before import chunk');
await bulkDataService.bulkDataInsert({ try {
projectName: projectId, await bulkDataService.bulkDataInsert({
tableName: modelId, projectName: projectId,
body: chunk, tableName: modelId,
cookie: null, body: chunk,
chunkSize: 1000, cookie: null,
foreign_key_checks: false chunkSize: chunk.length + 1,
}); foreign_key_checks: false
});
} catch (e) {
debugLog(`${model.title} import throwed an error!`);
console.log(e);
}
chunk = []; chunk = [];
elapsed_time('after chunk'); elapsedTime('after import chunk');
parser.resume(); parser.resume();
} }
} }
@ -685,16 +696,22 @@ export async function importBase(param: {
}, },
complete: async function () { complete: async function () {
if (chunk.length > 0) { if (chunk.length > 0) {
elapsed_time('before chunk'); elapsedTime('before import chunk');
await bulkDataService.bulkDataInsert({ try {
projectName: projectId, await bulkDataService.bulkDataInsert({
tableName: modelId, projectName: projectId,
body: chunk, tableName: modelId,
cookie: null, body: chunk,
foreign_key_checks: false cookie: null,
}); chunkSize: chunk.length + 1,
foreign_key_checks: false
});
} catch (e) {
debugLog(chunk);
console.log(e);
}
chunk = []; chunk = [];
elapsed_time('after chunk'); elapsedTime('after import chunk');
} }
resolve(null); resolve(null);
}, },
@ -702,6 +719,9 @@ export async function importBase(param: {
}); });
} }
// reset timer
elapsedTime();
for (const file of linkFiles) { for (const file of linkFiles) {
const readStream = await storageAdapter.fileReadByStream( const readStream = await storageAdapter.fileReadByStream(
`${path}/data/${file}` `${path}/data/${file}`
@ -719,7 +739,7 @@ export async function importBase(param: {
let pkIndex = -1; let pkIndex = -1;
console.log(`Linking ${model.title}...`); debugLog(`Linking ${model.title}...`);
await new Promise(async (resolve) => { await new Promise(async (resolve) => {
papaparse.parse(readStream, { papaparse.parse(readStream, {
@ -788,7 +808,8 @@ export async function importBase(param: {
}, },
complete: async function () { complete: async function () {
for (const [k, v] of Object.entries(chunk)) { for (const [k, v] of Object.entries(chunk)) {
try { try {
elapsedTime('prepare link chunk');
await bulkDataService.bulkDataInsert({ await bulkDataService.bulkDataInsert({
projectName: projectId, projectName: projectId,
tableName: k, tableName: k,
@ -797,8 +818,8 @@ export async function importBase(param: {
chunkSize: 1000, chunkSize: 1000,
foreign_key_checks: false foreign_key_checks: false
}); });
elapsedTime('insert link chunk');
} catch (e) { } catch (e) {
console.log('linkError');
console.log(e); console.log(e);
} }
} }

Write Preview
Loading…
Cancel
Save