From b127e4d4890350469329e668eb14888eae61fbfa Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Jun 2022 16:24:30 +0530 Subject: [PATCH] fix: handle LTAR alias contains `.` Signed-off-by: Pranav C --- .../src/lib/noco/meta/api/sync/helpers/job.ts | 189 ++---------------- .../api/sync/helpers/readAndProcessData.ts | 18 +- 2 files changed, 25 insertions(+), 182 deletions(-) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index 2bc1dfd46c..ea8082a276 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -83,6 +83,12 @@ export default async ( const ncLinkDataStore: any = {}; const insertedAssocRef: any = {}; + const atNcAliasRef: { + [ncTableId: string]: { + [ncTitle: string]: string; + }; + } = {}; + const uniqueTableNameGen = getUniqueNameGenerator('sheet'); // run time counter (statistics) @@ -619,6 +625,10 @@ export default async ( aTblLinkColumns[i].name ); + // LTAR alias ref to AT + atNcAliasRef[srcTbl.id] = atNcAliasRef[srcTbl.id] || {}; + atNcAliasRef[srcTbl.id][ncName.title] = aTblLinkColumns[i].name; + logDetailed( `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` ); @@ -1200,74 +1210,6 @@ export default async ( ////////// Data processing - // @ts-ignore - async function nocoLinkProcessing(projName, table, record, _field) { - const rec = record.fields; - - for (const [key, value] of Object.entries(rec)) { - const refRowIdList: any = value; - const referenceColumnName = key; - - if (refRowIdList.length) { - for (let i = 0; i < refRowIdList.length; i++) { - logDetailed( - `NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}` - ); - - const _perfStart = recordPerfStart(); - await api.dbTableRow.nestedAdd( - 'noco', - projName, - table.id, - `${record.id}`, - 'mm', - encodeURIComponent(referenceColumnName), - `${refRowIdList[i]}` - ); - recordPerfStats(_perfStart, 'dbTableRow.nestedAdd'); - } - } - } - } - // - // ////////// Data processing - // async function nocoLinkProcessingv2( - // projName, - // table, - // record, - // _field, - // rowsRef = {} - // ) { - // const modelMeta = await api.dbTable.read(table.id); - // - // const rec = record.fields; - // - // for (const [key, value] of Object.entries(rec)) { - // const refRowIdList: any = value; - // const referenceColumnName = key; - // - // if (refRowIdList.length) { - // for (let i = 0; i < refRowIdList.length; i++) { - // logDetailed( - // `NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}` - // ); - // - // const _perfStart = recordPerfStart(); - // await api.dbTableRow.nestedAdd( - // 'noco', - // projName, - // table.id, - // `${record.id}`, - // 'mm',0 - // encodeURIComponent(referenceColumnName), - // `${refRowIdList[i]}` - // ); - // recordPerfStats(_perfStart, 'dbTableRow.nestedAdd'); - // } - // } - // } - // } - async function nocoBaseDataProcessing_v2(sDB, table, record) { const recordHash = hash(record); const rec = { ...record.fields }; @@ -1367,48 +1309,6 @@ export default async ( if (!syncDB.options.syncAttachment) rec[key] = null; else { let tempArr = []; - // for (const v of value) { - // const binaryImage = await axios - // .get(v.url, { - // responseType: 'stream', - // headers: { - // 'Content-Type': v.type - // } - // }) - // .then(response => { - // return response.data; - // }) - // .catch(error => { - // console.log(error); - // return false; - // }); - // - // const imageFile: any = new FormData(); - // imageFile.append('files', binaryImage, { - // filename: v.filename.includes('?') - // ? v.filename.split('?')[0] - // : v.filename - // }); - // - // const rs = await axios - // .post(sDB.baseURL + '/api/v1/db/storage/upload', imageFile, { - // params: { - // path: `noco/${sDB.projectName}/${table.title}/${key}` - // }, - // headers: { - // 'Content-Type': `multipart/form-data; boundary=${imageFile._boundary}`, - // 'xc-auth': sDB.authToken - // } - // }) - // .then(response => { - // return response.data; - // }) - // .catch(e => { - // console.log(e); - // }); - // - // tempArr.push(...(rs || [])); - // } try { tempArr = await api.storage.uploadByUrl( @@ -1442,72 +1342,6 @@ export default async ( return rec; } - // @ts-ignore - async function nocoReadData(sDB, table) { - ncLinkDataStore[table.title] = {}; - const insertJobs: Promise[] = []; - - return new Promise((resolve, reject) => { - base(table.title) - .select({ - pageSize: 100 - // maxRecords: 1, - }) - .eachPage( - async function page(records, fetchNextPage) { - // console.log(JSON.stringify(records, null, 2)); - - // This function (`page`) will get called for each page of records. - logBasic( - `:: ${table.title} : ${recordCnt + 1} ~ ${(recordCnt += 100)}` - ); - - // await Promise.all( - // records.map(record => _callback(sDB, table, record)) - // ); - const ncRecords = []; - for (let i = 0; i < records.length; i++) { - const r = await nocoBaseDataProcessing_v2(sDB, table, records[i]); - ncRecords.push(r); - } - - // wait for previous job's to finish - await Promise.all(insertJobs); - - const _perfStart = recordPerfStart(); - insertJobs.push( - api.dbTableRow.bulkCreate( - 'nc', - sDB.projectName, - table.id, // encodeURIComponent(table.title), - ncRecords - ) - ); - recordPerfStats(_perfStart, 'dbTableRow.bulkCreate'); - - // To fetch the next page of records, call `fetchNextPage`. - // If there are more records, `page` will get called again. - // If there are no more records, `done` will get called. - // logBasic( - // `:: ${Date.now()} Awaiting response from Airtable Data API ...` - // ); - fetchNextPage(); - }, - async function done(err) { - if (err) { - console.error(err); - reject(err); - } - - // wait for all jobs to be completed - await Promise.all(insertJobs); - - resolve(null); - } - ); - }); - } - // @ts-ignore async function nocoReadDataSelected(projName, table, callback, fields) { return new Promise((resolve, reject) => { @@ -2301,7 +2135,8 @@ export default async ( logBasic, insertedAssocRef, logDetailed, - records: recordsMap[ncTbl.id] + records: recordsMap[ncTbl.id], + atNcAliasRef }); } diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts index fae6b13dbd..fc9d30b3fe 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts @@ -134,7 +134,8 @@ export async function importLTARData({ insertedAssocRef = {}, logDetailed = _str => {}, logBasic = _str => {}, - records + records, + atNcAliasRef }: { projectName: string; table: { title?: string; id?: string }; @@ -145,6 +146,11 @@ export async function importLTARData({ api: Api; insertedAssocRef: { [assocTableId: string]: boolean }; records?: Array<{ fields: any; id: string }>; + atNcAliasRef: { + [ncTableId: string]: { + [ncTitle: string]: string; + }; + }; }) { const assocTableMetas: Array<{ modelMeta: { id?: string; title?: string }; @@ -207,10 +213,12 @@ export async function importLTARData({ // todo: use actual alias instead of sanitized assocTableData.push( - ...(rec?.[assocMeta.colMeta.title] || []).map(id => ({ - [assocMeta.curCol.title]: record.id, - [assocMeta.refCol.title]: id - })) + ...(rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || []).map( + id => ({ + [assocMeta.curCol.title]: record.id, + [assocMeta.refCol.title]: id + }) + ) ); }