From fad224acaab0e9ac0066acdcddc9ef2f30bb95cb Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Sat, 28 May 2022 23:48:36 +0530 Subject: [PATCH] feat: cache for migration files Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../src/lib/noco/meta/api/sync/helpers/job.ts | 128 +++++++++++++----- 1 file changed, 95 insertions(+), 33 deletions(-) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index efdca96ffe..ef7406e909 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -83,6 +83,7 @@ export default async ( const storeLinks = false; const ncLinkDataStore: any = {}; const pageSize = 100; + const enableLocalCache = true; const uniqueTableNameGen = getUniqueNameGenerator('sheet'); @@ -124,7 +125,18 @@ export default async ( } else { await FetchAT.initialize(sDB.shareId); } - const ft = await FetchAT.read(); + + const filePath = `./migrationCache/meta/${sDB.shareId}.json`; + let ft; + if (enableLocalCache && fs2.existsSync(filePath)) { + ft = jsonfile.readFileSync(filePath); + } else { + ft = await FetchAT.read(); + if (enableLocalCache) + jsonfile.writeFileSync(filePath, ft, { + spaces: 2 + }); + } const duration = Date.now() - start; rtc.fetchAt.count++; rtc.fetchAt.time += duration; @@ -135,19 +147,28 @@ export default async ( // store copy of airtable schema globally g_aTblSchema = file.tableSchemas; - if (debugMode) jsonfile.writeFileSync('aTblSchema.json', ft, { spaces: 2 }); - return file; } async function getViewData(viewId) { const start = Date.now(); - const ft = await FetchAT.readView(viewId); + + const filePath = `./migrationCache/meta/${viewId}.json`; + let ft; + if (enableLocalCache && fs2.existsSync(filePath)) { + ft = jsonfile.readFileSync(filePath); + } else { + ft = await FetchAT.readView(viewId); + + if (enableLocalCache) + jsonfile.writeFileSync(filePath, ft, { + spaces: 2 + }); + } const duration = Date.now() - start; rtc.fetchAt.count++; rtc.fetchAt.time += duration; - if (debugMode) jsonfile.writeFileSync(`${viewId}.json`, ft, { spaces: 2 }); return ft.view; } @@ -1316,6 +1337,7 @@ export default async ( case UITypes.Attachment: if (syncDB.options.syncLookup) rec[key] = null; else { + console.log(`:: Attach : ${record.id}, ${key}, ${value}`); const tempArr = []; for (const v of value) { const binaryImage = await axios @@ -1379,11 +1401,61 @@ export default async ( ncLinkDataStore[table.title] = {}; const insertJobs: Promise[] = []; + async function processRecords(records) { + // This function (`page`) will get called for each page of records. + logBasic( + `:: ${table.title} : ${recordCnt + 1} ~ ${(recordCnt += pageSize)}` + ); + + // await Promise.all( + // records.map(record => _callback(sDB, table, record)) + // ); + const ncRecords = []; + for (let i = 0; i < records.length; i++) { + const r = await nocoBaseDataProcessing_v2(sDB, table, records[i]); + ncRecords.push(r); + } + + // wait for previous job's to finish + await Promise.all(insertJobs); + + const _perfStart = recordPerfStart(); + insertJobs.push( + api.dbTableRow.bulkCreate( + 'nc', + sDB.projectName, + table.id, // encodeURIComponent(table.title), + ncRecords + ) + ); + recordPerfStats(_perfStart, 'dbTableRow.bulkCreate'); + } + // skip virtual columns const fieldsArray = aTbl.columns .filter(a => !['formula', 'lookup', 'rollup'].includes(a.type)) .map(a => a.name); + if (enableLocalCache) { + // check if cached file exists for this table + const f = `./migrationCache/data/${table.title}_${recordCnt / + pageSize}.json`; + + while (fs2.existsSync(f)) { + const records = jsonfile.readFileSync(f); + logBasic( + `:: ${table.title} : ${recordCnt + 1} ~ ${(recordCnt += pageSize)}` + ); + await processRecords(records); + } + + // scenarios to handle + // - cache file doesn't exist + // - previous cache read was partial + + return; + } + return new Promise((resolve, reject) => { base(table.title) .select({ @@ -1393,36 +1465,18 @@ export default async ( }) .eachPage( async function page(records, fetchNextPage) { + if (enableLocalCache) + jsonfile.writeFileSync( + `./migrationCache/data/${table.title}_${recordCnt / + pageSize}.json`, + records, + { + spaces: 2 + } + ); // console.log(JSON.stringify(records, null, 2)); - // This function (`page`) will get called for each page of records. - logBasic( - `:: ${table.title} : ${recordCnt + - 1} ~ ${(recordCnt += pageSize)}` - ); - - // await Promise.all( - // records.map(record => _callback(sDB, table, record)) - // ); - const ncRecords = []; - for (let i = 0; i < records.length; i++) { - const r = await nocoBaseDataProcessing_v2(sDB, table, records[i]); - ncRecords.push(r); - } - - // wait for previous job's to finish - await Promise.all(insertJobs); - - const _perfStart = recordPerfStart(); - insertJobs.push( - api.dbTableRow.bulkCreate( - 'nc', - sDB.projectName, - table.id, // encodeURIComponent(table.title), - ncRecords - ) - ); - recordPerfStats(_perfStart, 'dbTableRow.bulkCreate'); + await processRecords(records); // To fetch the next page of records, call `fetchNextPage`. // If there are more records, `page` will get called again. @@ -2102,6 +2156,9 @@ export default async ( /////////////////////////////////////////////////////////////////////////////// let recordCnt = 0; + const fs2 = require('fs'); + const dir = ['./migrationCache/meta', './migrationCache/data']; + try { logBasic('SDK initialized'); api = new Api({ @@ -2111,6 +2168,11 @@ export default async ( } }); + if (!fs2.existsSync(dir)) { + fs2.mkdirSync(dir[0], { recursive: true }); + fs2.mkdirSync(dir[1], { recursive: true }); + } + logDetailed('Project initialization started'); // delete project if already exists if (debugMode) await init(syncDB);