From 7eed0d3cafaad1199f26f02ac564c4022b3c5acd Mon Sep 17 00:00:00 2001 From: mertmit Date: Thu, 28 Apr 2022 03:39:20 +0300 Subject: [PATCH] sync: typescript migration Signed-off-by: mertmit --- packages/nocodb/tests/sync/ts/attachment.json | 337 ++++++++ packages/nocodb/tests/sync/ts/src/nc-sync.ts | 761 ++++++++++++++++++ packages/nocodb/tests/sync/ts/tsconfig.json | 63 ++ scripts/sdk/swagger.json | 2 +- 4 files changed, 1162 insertions(+), 1 deletion(-) create mode 100644 packages/nocodb/tests/sync/ts/attachment.json create mode 100644 packages/nocodb/tests/sync/ts/src/nc-sync.ts create mode 100644 packages/nocodb/tests/sync/ts/tsconfig.json diff --git a/packages/nocodb/tests/sync/ts/attachment.json b/packages/nocodb/tests/sync/ts/attachment.json new file mode 100644 index 0000000000..8274ab64f7 --- /dev/null +++ b/packages/nocodb/tests/sync/ts/attachment.json @@ -0,0 +1,337 @@ +{ + "appBlanket": { + "userInfoById": { + "usrcTFn14vKTIgbW3": { + "id": "usrcTFn14vKTIgbW3", + "firstName": "Steyer", + "lastName": "Rom", + "email": "steyerrom@gmail.com", + "profilePicUrl": "https://static.airtable.com/images/userIcons/user_icon_9.png", + "permissionLevel": "owner", + "appBlanketUserState": "active" + } + }, + "externalAccountInfoById": {}, + "userGroupInfoById": {}, + "workspaceSyncSources": [], + "activeUserIdByAcceptedInviteId": {}, + "isWorkspaceOptedOutOfUserContentCdnAuth": false, + "isEnterpriseAccountOptedOutOfUserContentCdnAuth": false, + "enterpriseAttachmentRestrictions": { + "restrictionType": "unrestricted", + "attachmentTypeAllowlist": [] + }, + "isWorkspaceLinkedToEnterpriseAccount": false + }, + "description": null, + "sortTiebreakerKey": "appEHTLsc4lSaia9A", + "defaultViewMutability": null, + "maintenanceModeSettings": null, + "sharesById": { + "shrqM5QS9sSZ94mQx": { + "id": "shrqM5QS9sSZ94mQx", + "modelId": "appEHTLsc4lSaia9A", + "createdByUserId": "usrcTFn14vKTIgbW3", + "canBeCloned": false, + "canBeExported": false, + "includeHiddenColumns": false, + "includeBlocks": true, + "emailDomain": null, + "hasPassword": false, + "generationNumber": 0, + "metadata": null + } + }, + "workflowSectionsById": {}, + "applicationTransactionNumber": 21, + "tableSchemas": [ + { + "id": "tblXYuhMZ3hWZkBCa", + "name": "Table 1", + "primaryColumnId": "fldrhmH0EYnOXfnUA", + "columns": [ + { + "id": "fldrhmH0EYnOXfnUA", + "name": "Name", + "type": "text" + }, + { + "id": "fldVFoK7t6aC92xzj", + "name": "Notes", + "type": "multilineText" + }, + { + "id": "fld66bK6Pq8AG4m3h", + "name": "Attachments", + "type": "multipleAttachment", + "typeOptions": { + "unreversed": true + } + }, + { + "id": "fldRC8zKoyGzM6agG", + "name": "Status", + "type": "select", + "typeOptions": { + "choices": { + "selQSYarqhTyVrwZw": { + "id": "selQSYarqhTyVrwZw", + "name": "Todo", + "color": "red" + }, + "selOFyoifOyV50QI7": { + "id": "selOFyoifOyV50QI7", + "name": "In progress", + "color": "yellow" + }, + "selsxseijq4XvcTB8": { + "id": "selsxseijq4XvcTB8", + "name": "Done", + "color": "green" + } + }, + "choiceOrder": [ + "selQSYarqhTyVrwZw", + "selOFyoifOyV50QI7", + "selsxseijq4XvcTB8" + ] + } + } + ], + "meaningfulColumnOrder": [ + { + "columnId": "fldrhmH0EYnOXfnUA", + "visibility": true + }, + { + "columnId": "fldVFoK7t6aC92xzj", + "visibility": true + }, + { + "columnId": "fld66bK6Pq8AG4m3h", + "visibility": true + }, + { + "columnId": "fldRC8zKoyGzM6agG", + "visibility": true + } + ], + "views": [ + { + "id": "viwbgKWGvUoZCosF1", + "name": "Grid view", + "type": "grid", + "createdByUserId": "usrcTFn14vKTIgbW3" + } + ], + "viewOrder": [ + "viwbgKWGvUoZCosF1" + ], + "viewsById": { + "viwbgKWGvUoZCosF1": { + "id": "viwbgKWGvUoZCosF1", + "name": "Grid view", + "type": "grid", + "createdByUserId": "usrcTFn14vKTIgbW3" + } + }, + "viewSectionsById": {}, + "schemaChecksum": "412180368d81674e723b957501f16a57c9264fc69d19668b70d1547888c29413" + } + ], + "tableDatas": [ + { + "id": "tblXYuhMZ3hWZkBCa", + "rows": [ + { + "id": "recv9Z8uFFNt50rqX", + "createdTime": "2022-04-27T18:48:37.000Z", + "cellValuesByColumnId": { + "fld66bK6Pq8AG4m3h": [ + { + "id": "attdRIU80oCDC8u6X", + "url": "https://dl.airtable.com/.attachments/247a5881e7742c2d55cb8f814fe7263a/964d1018/512x512.png", + "filename": "512x512.png", + "type": "image/png", + "size": 77822, + "width": 2763, + "height": 2763, + "smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/f5f0717d00f1f61c402fec203d16efd7/61bea79e", + "smallThumbWidth": 36, + "smallThumbHeight": 36, + "largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/8f49ea3dcdaf3aa9788e84f1a3e3f3e2/81284101", + "largeThumbWidth": 512, + "largeThumbHeight": 512, + "fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/c5300e8cda92de966de82c760cd44533/89d51367", + "fullThumbWidth": 3000, + "fullThumbHeight": 3000 + } + ], + "fldrhmH0EYnOXfnUA": "nc" + } + }, + { + "id": "recmAktd3OQe3Wg8C", + "createdTime": "2022-04-27T18:48:37.000Z", + "cellValuesByColumnId": { + "fld66bK6Pq8AG4m3h": [ + { + "id": "attLBB2eqE9grLlUU", + "url": "https://dl.airtable.com/.attachments/a67aaa1efa29d40c3633ca03f0b366e6/9b56c6e2/Abstract-Nord.png", + "filename": "Abstract-Nord.png", + "type": "image/png", + "size": 140219, + "width": 1920, + "height": 1080, + "smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/d5daa7de864a8302ad9d2c43a257a52d/2b709aa8", + "smallThumbWidth": 64, + "smallThumbHeight": 36, + "largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/7b388a8484a1bc8f9f91bbf304c9b42e/4663133b", + "largeThumbWidth": 910, + "largeThumbHeight": 512, + "fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/4d5dfa21efa34198c740443551531eea/ea51b594", + "fullThumbWidth": 3000, + "fullThumbHeight": 3000 + }, + { + "id": "attIzZpRBBWwRI5Wz", + "url": "https://dl.airtable.com/.attachments/107f2f1d886b1f1ae0d528ba7f76df03/931109ad/archlinux.png", + "filename": "archlinux.png", + "type": "image/png", + "size": 184887, + "width": 3840, + "height": 2160, + "smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/cb9894ae30f7a264a87639d5ce4980ec/e000fb24", + "smallThumbWidth": 64, + "smallThumbHeight": 36, + "largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/1f4146a90346325a8ab51d5d7f4f40be/9a73928f", + "largeThumbWidth": 910, + "largeThumbHeight": 512, + "fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/9954669a1ba7bc2d4e4d50f9c0130435/7d46d64a", + "fullThumbWidth": 3000, + "fullThumbHeight": 3000 + }, + { + "id": "atttj7OPAZ1iDtHBt", + "url": "https://dl.airtable.com/.attachments/4eec95e15a951829c2b12e4375bece7f/8ff26ede/arctic-landscape.png", + "filename": "arctic-landscape.png", + "type": "image/png", + "size": 155548, + "width": 1920, + "height": 1080, + "smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/dd63a0d18ed1c4bf3e7096145fc5fa0f/ebfc95ac", + "smallThumbWidth": 64, + "smallThumbHeight": 36, + "largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/c9b441a88304b53cdc44fa0fcfdb57e6/8c6b629d", + "largeThumbWidth": 910, + "largeThumbHeight": 512, + "fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/f5b74f24a31f22e8e859954499abfa67/fd11b916", + "fullThumbWidth": 3000, + "fullThumbHeight": 3000 + } + ], + "fldrhmH0EYnOXfnUA": "wp" + } + }, + { + "id": "recAFj2eQVsynoDJa", + "createdTime": "2022-04-27T18:48:37.000Z", + "cellValuesByColumnId": { + "fldrhmH0EYnOXfnUA": "test" + } + } + ], + "viewDatas": [ + { + "id": "viwbgKWGvUoZCosF1", + "frozenColumnCount": 1, + "columnOrder": [ + { + "columnId": "fldrhmH0EYnOXfnUA", + "visibility": true + }, + { + "columnId": "fldVFoK7t6aC92xzj", + "visibility": true + }, + { + "columnId": "fld66bK6Pq8AG4m3h", + "visibility": true + }, + { + "columnId": "fldRC8zKoyGzM6agG", + "visibility": true + } + ], + "sharesById": {}, + "createdByUserId": "usrcTFn14vKTIgbW3", + "applicationTransactionNumber": 21, + "rowOrder": [ + { + "rowId": "recv9Z8uFFNt50rqX", + "visibility": true + }, + { + "rowId": "recmAktd3OQe3Wg8C", + "visibility": true + }, + { + "rowId": "recAFj2eQVsynoDJa", + "visibility": true + } + ] + } + ], + "hasOnlyIncludedRowAndCellDataForIncludedViews": false + } + ], + "hasBlockInstallations": false, + "applicationAdminFlags": { + "UPDATE_PRIMITIVE_CELL_THROTTLE_MS": null, + "MAX_WORKFLOWS_PER_APPLICATION": null, + "MAX_SYNC_SOURCES_PER_APPLICATION": null, + "MAX_SYNC_SOURCES_PER_TABLE": null, + "MAX_SYNCED_TABLES_PER_APPLICATION": null, + "CUSTOM_MAX_NUM_ROWS_PER_TABLE": null + }, + "pageBundles": [], + "uploadedUserContentCdnSetting": { + "applicationScopedAuthMode": "public" + }, + "applicationV2TargetedFeatureFlagClientConfiguration": { + "nonCollaboratorsInCollaboratorField": { + "trafficLevel": 0 + }, + "applicationInsights": { + "trafficLevel": 0 + }, + "autoOpenInsightsPaneOnUnseenSuggestion": { + "trafficLevel": 0 + }, + "disabledWorkflowOnSchemaChangeSuggestion": { + "trafficLevel": 0 + }, + "syncFailureSuggestion": { + "trafficLevel": 0 + }, + "unusedViewsSuggestion": { + "trafficLevel": 0 + }, + "filterUnusedViewsUsingDependencyGraph": { + "trafficLevel": 100 + }, + "unusedSelectChoicesSuggestion": { + "trafficLevel": 0 + }, + "unifiedEventLog": { + "trafficLevel": 0 + }, + "constantPoolingForCrudResponses": { + "trafficLevel": 0 + } + }, + "applicationV2EnabledFeatureNames": [ + "filterUnusedViewsUsingDependencyGraph" + ], + "isConstantPooledData": false +} \ No newline at end of file diff --git a/packages/nocodb/tests/sync/ts/src/nc-sync.ts b/packages/nocodb/tests/sync/ts/src/nc-sync.ts new file mode 100644 index 0000000000..254eeded3c --- /dev/null +++ b/packages/nocodb/tests/sync/ts/src/nc-sync.ts @@ -0,0 +1,761 @@ +import { Api, UITypes } from 'nocodb-sdk'; +import Airtable from 'airtable'; +import jsonfile from 'jsonfile'; +import FormData from 'form-data' +import axios from 'axios'; + + +//RUN: npx ts-node src/nc-sync.ts + +function syncLog(log) { + console.log(`nc-sync: ${log}`) +} + +// apiKey & baseID configurations required to read data using Airtable APIs +// +const syncDB = { + airtable: { + apiKey: 'keyfaOQmPOpigyJV8', + baseId: 'appEHTLsc4lSaia9A', + schemaJson: 'attachment.json' + }, + projectName: 'sample', + baseURL: 'http://localhost:8080', + authToken: + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InRlc3RAdGVzdC50ZXN0IiwiZmlyc3RuYW1lIjpudWxsLCJsYXN0bmFtZSI6bnVsbCwiaWQiOiJ1c183eHJ5b25jYWZzbHd2diIsInJvbGVzIjoidXNlcixzdXBlciIsImlhdCI6MTY1MDcxMjMyN30.zB_E46qkQy1mCqjjJL89WPa1jCY101BAAoLAyE7b1n8' +}; + +const api = new Api({ + baseURL: syncDB.baseURL, + headers: { + 'xc-auth': syncDB.authToken + } +}); + +// global schema store +let global_schema: any = getAtableSchema().tableSchemas; + +function getAtableSchema() { + return jsonfile.readFileSync(syncDB.airtable.schemaJson); +} + +// base mapping table +const aTblNcTypeMap = { + foreignKey: UITypes.LinkToAnotherRecord, + text: UITypes.SingleLineText, + multilineText: UITypes.LongText, + multipleAttachment: UITypes.Attachment, + checkbox: UITypes.Checkbox, + multiSelect: UITypes.MultiSelect, + select: UITypes.SingleSelect, + collaborator: UITypes.Collaborator, + date: UITypes.Date, + // kludge: phone: UITypes.PhoneNumber, + phone: UITypes.SingleLineText, + number: UITypes.Number, + rating: UITypes.Rating, + // kludge: formula: UITypes.Formula, + formula: UITypes.SingleLineText, + rollup: UITypes.Rollup, + count: UITypes.Count, + lookup: UITypes.Lookup, + autoNumber: UITypes.AutoNumber, + barcode: UITypes.Barcode, + button: UITypes.Button +}; + +//----------------------------------------------------------------------------- +// aTbl helper routines +// + +// aTbl: retrieve table name from table ID +// +function aTbl_getTableName(tblId) { + const sheetObj = global_schema.find(tbl => tbl.id === tblId); + return { + tn: sheetObj.name + }; +} + +// aTbl: retrieve column name from column ID +// +function aTbl_getColumnName(colId) { + for (let i = 0; i < global_schema.length; i++) { + let sheetObj = global_schema[i]; + const column = sheetObj.columns.find(col => col.id === colId); + if (column !== undefined) + return { + tn: sheetObj.name, + cn: column.name + }; + } + return {}; +} + +// nc dump schema +/* +async function nc_DumpTableSchema() { + console.log('['); + let ncTblList = await api.dbTable.list(global_ncCreatedProjectSchema.id); + for (let i = 0; i < ncTblList.list.length; i++) { + let ncTbl = await api.dbTable.read(ncTblList.list[i].id); + console.log(JSON.stringify(ncTbl, null, 2)); + console.log(','); + } + console.log(']'); +} +*/ + +// retrieve nc column schema from using aTbl field ID as reference +// +async function nc_getColumnSchema(aTblFieldId) { + let ncTblList = await api.dbTable.list(global_ncCreatedProjectSchema.id); + let aTblField = aTbl_getColumnName(aTblFieldId); + let ncTblId = ncTblList.list.filter(x => x.title === aTblField.tn)[0].id; + let ncTbl = await api.dbTable.read(ncTblId); + let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn); + return ncCol; +} + +// retrieve nc table schema using table name +async function nc_getTableSchema(tableName) { + let ncTblList = await api.dbTable.list(global_ncCreatedProjectSchema.id); + let ncTblId = ncTblList.list.filter(x => x.title === tableName)[0].id; + let ncTbl = await api.dbTable.read(ncTblId); + return ncTbl; +} + +// delete project if already exists +async function init() { + // delete 'sample' project if already exists + let x = await api.project.list() + + let sampleProj = x.list.find(a => a.title === syncDB.projectName) + if (sampleProj) { + await api.project.delete(sampleProj.id) + } + + syncLog('Init') +} + +// map UIDT +// +function getNocoType(col) { + // start with default map + let ncType = aTblNcTypeMap[col.type]; + + // types email & url are marked as text + // types currency & percent, duration are marked as number + // types createTime & modifiedTime are marked as formula + + switch (col.type) { + case 'text': + if (col.typeOptions?.validatorName === 'email') ncType = UITypes.Email; + else if (col.typeOptions?.validatorName === 'url') ncType = UITypes.URL; + break; + + case 'number': + // kludge: currency validation error with decimal places + if (col.typeOptions?.format === 'percentV2') ncType = UITypes.Percent; + else if (col.typeOptions?.format === 'duration') + ncType = UITypes.Duration; + else if (col.typeOptions?.format === 'currency') + ncType = UITypes.Currency; + break; + + case 'formula': + if (col.typeOptions?.formulaTextParsed === 'CREATED_TIME()') + ncType = UITypes.CreateTime; + else if (col.typeOptions?.formulaTextParsed === 'LAST_MODIFIED_TIME()') + ncType = UITypes.LastModifiedTime; + break; + } + + return ncType; +} + +// retrieve additional options associated with selected data types +// +function getNocoTypeOptions(col) { + switch (col.type) { + case 'select': + case 'multiSelect': + // prepare options list in CSV format + // note: NC doesn't allow comma's in options + // + let opt = []; + for (let [, value] of Object.entries(col.typeOptions.choices) as any) { + opt.push(value.name); + } + let csvOpt = "'" + opt.join("','") + "'"; + return { type: 'select', data: csvOpt }; + + default: + return { type: undefined }; + } +} + +// convert to Nc schema (basic, excluding relations) +// +function tablesPrepare(tblSchema) { + let tables = []; + for (let i = 0; i < tblSchema.length; ++i) { + let table: any = {}; + + syncLog(`Preparing base schema (sans relations): ${tblSchema[i].name}`) + + // table name + table.table_name = tblSchema[i].name; + table.title = tblSchema[i].name; + + // insert record_id of type ID by default + table.columns = [ + { + title: 'record_id', + column_name: 'record_id', + uidt: UITypes.ID + // uidt: UITypes.SingleLineText, + // pk: true + } + ]; + + for (let j = 0; j < tblSchema[i].columns.length; j++) { + let col = tblSchema[i].columns[j]; + + // skip link, lookup, rollup fields in this iteration + if (['foreignKey', 'lookup', 'rollup'].includes(col.type)) continue; + + // not supported datatype + // if (['formula'].includes(col.type)) continue; + + // base column schema + // kludge: error observed in Nc with space around column-name + let ncCol: any = { + title: col.name.trim(), + + // knex complains use of '?' in field name + //column_name: col.name.replace(/\?/g, '\\?').trim(), + column_name: col.name.replace(/\?/g, 'QQ').trim(), + uidt: getNocoType(col) + }; + + // additional column parameters when applicable + let colOptions = getNocoTypeOptions(col); + + switch (colOptions.type) { + case 'select': + ncCol.dtxp = colOptions.data; + break; + + case undefined: + break; + } + + table.columns.push(ncCol); + } + + tables.push(table); + } + return tables; +} + +async function nocoCreateBaseSchema(srcSchema) { + // base schema preparation: exclude + let tables = tablesPrepare(srcSchema.tableSchemas); + + // for each table schema, create nc table + for (let idx = 0; idx < tables.length; idx++) { + + syncLog(`dbTable.create ${tables[idx].title}`) + let table = await api.dbTable.create( + global_ncCreatedProjectSchema.id, + tables[idx] + ); + } + + // debug + // console.log(JSON.stringify(tables, null, 2)); + return tables; +} + +async function nocoCreateLinkToAnotherRecord(aTblSchema) { + // Link to another RECORD + for (let idx = 0; idx < aTblSchema.length; idx++) { + let aTblLinkColumns = aTblSchema[idx].columns.filter( + x => x.type === 'foreignKey' + ); + + // Link columns exist + // + if (aTblLinkColumns.length) { + for (let i = 0; i < aTblLinkColumns.length; i++) { + + { + let src = aTbl_getColumnName(aTblLinkColumns[i].id) + let dst = aTbl_getColumnName(aTblLinkColumns[i].typeOptions.symmetricColumnId) + syncLog(` LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst.tn}:${dst.cn}`) + } + + // check if link already established? + if (!nc_isLinkExists(aTblLinkColumns[i].id)) { + // parent table ID + let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + + // find child table name from symmetric column ID specified + let childTable = aTbl_getColumnName( + aTblLinkColumns[i].typeOptions.symmetricColumnId + ); + + // retrieve child table ID (nc) from table name + let childTableId = (await nc_getTableSchema(childTable.tn)).id; + + // create link + let column = await api.dbTableColumn.create(srcTableId, { + uidt: 'LinkToAnotherRecord', + title: aTblLinkColumns[i].name, + parentId: srcTableId, + childId: childTableId, + type: 'mm' + // aTblLinkColumns[i].typeOptions.relationship === 'many' + // ? 'mm' + // : 'hm' + }); + syncLog(`NC API: dbTableColumn.create LinkToAnotherRecord`) + + // store link information in separate table + // this information will be helpful in identifying relation pair + let link = { + nc: { + title: aTblLinkColumns[i].name, + parentId: srcTableId, + childId: childTableId, + type: 'mm' + }, + aTbl: { + tblId: aTblSchema[idx].id, + ...aTblLinkColumns[i] + } + }; + + global_ncLinkMappingTable.push(link); + } else { + // if link already exists, we need to change name of linked column + // to what is represented in airtable + + // 1. extract associated link information from link table + // 2. retrieve parent table information (source) + // 3. using foreign parent & child column ID, find associated mapping in child table + // 4. update column name + let x = global_ncLinkMappingTable.findIndex( + x => + x.aTbl.tblId === aTblLinkColumns[i].typeOptions.foreignTableId && + x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId + ); + + let childTblSchema = await api.dbTable.read( + global_ncLinkMappingTable[x].nc.childId + ); + let parentTblSchema = await api.dbTable.read( + global_ncLinkMappingTable[x].nc.parentId + ); + + let parentLinkColumn: any = parentTblSchema.columns.find( + col => col.title === global_ncLinkMappingTable[x].nc.title + ); + + let childLinkColumn: any = {}; + if (parentLinkColumn.colOptions.type == 'hm') { + // for hm: + // mapping between child & parent column id is direct + // + childLinkColumn = childTblSchema.columns.find( + (col: any) => + col.uidt === 'LinkToAnotherRecord' && + col.colOptions.fk_child_column_id === + parentLinkColumn.colOptions.fk_child_column_id && + col.colOptions.fk_parent_column_id === + parentLinkColumn.colOptions.fk_parent_column_id + ); + } else { + // for mm: + // mapping between child & parent column id is inverted + // + childLinkColumn = childTblSchema.columns.find( + (col: any) => + col.uidt === 'LinkToAnotherRecord' && + col.colOptions.fk_child_column_id === + parentLinkColumn.colOptions.fk_parent_column_id && + col.colOptions.fk_parent_column_id === + parentLinkColumn.colOptions.fk_child_column_id && + col.colOptions.fk_mm_model_id === + parentLinkColumn.colOptions.fk_mm_model_id + ); + } + + // rename + // note that: current rename API requires us to send all parameters, + // not just title being renamed + let res = await api.dbTableColumn.update(childLinkColumn.id, { + ...childLinkColumn, + title: aTblLinkColumns[i].name, + }); + // console.log(res.columns.find(x => x.title === aTblLinkColumns[i].name)) + syncLog(`dbTableColumn.update rename symmetric column`) + } + } + } + } +} + +async function nocoCreateLookups(aTblSchema) { + // LookUps + for (let idx = 0; idx < aTblSchema.length; idx++) { + let aTblColumns = aTblSchema[idx].columns.filter(x => x.type === 'lookup'); + + // parent table ID + let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + + if (aTblColumns.length) { + // Lookup + for (let i = 0; i < aTblColumns.length; i++) { + let ncRelationColumn = await nc_getColumnSchema( + aTblColumns[i].typeOptions.relationColumnId + ); + let ncLookupColumn = await nc_getColumnSchema( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + + let lookupColumn = await api.dbTableColumn.create(srcTableId, { + uidt: 'Lookup', + title: aTblColumns[i].name, + fk_relation_column_id: ncRelationColumn.id, + fk_lookup_column_id: ncLookupColumn.id + }); + + syncLog(`NC API: dbTableColumn.create LOOKUP`) + } + } + } +} + +async function nocoCreateRollups(aTblSchema) { + // Rollups + for (let idx = 0; idx < aTblSchema.length; idx++) { + let aTblColumns = aTblSchema[idx].columns.filter(x => x.type === 'rollup'); + + // parent table ID + let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; + + if (aTblColumns.length) { + // rollup exist + for (let i = 0; i < aTblColumns.length; i++) { + let ncRelationColumn = await nc_getColumnSchema( + aTblColumns[i].typeOptions.relationColumnId + ); + let ncRollupColumn = await nc_getColumnSchema( + aTblColumns[i].typeOptions.foreignTableRollupColumnId + ); + + let lookupColumn = await api.dbTableColumn.create(srcTableId, { + uidt: 'Rollup', + title: aTblColumns[i].name, + fk_relation_column_id: ncRelationColumn.id, + fk_rollup_column_id: ncRollupColumn.id, + rollup_function: 'sum' // fix me: hardwired + }); + + syncLog(`NC API: dbTableColumn.create ROLLUP`) + + } + } + } +} + +async function nocoSetPrimary(aTblSchema) { + for (let idx = 0; idx < aTblSchema.length; idx++) { + let pColId = aTblSchema[idx].primaryColumnId; + let ncCol = await nc_getColumnSchema(pColId); + + syncLog(`NC API: dbTableColumn.primaryColumnSet`) + await api.dbTableColumn.primaryColumnSet(ncCol.id); + } +} + +////////// Data processing + +// https://www.airtable.com/app1ivUy7ba82jOPn/api/docs#javascript/metadata +let base = new Airtable({ apiKey: syncDB.airtable.apiKey }).base( + syncDB.airtable.baseId +); + +let aTblDataLinks = []; +let aTblNcRecordMappingTable = {}; + +function nocoLinkProcessing(table, record, field) { + (async () => { + let rec = record.fields; + const value = Object.values(rec) as any; + let srcRow = aTblNcRecordMappingTable[`${record.id}`]; + + if (value.length) { + for (let i = 0; i < value[0].length; i++) { + let dstRow = aTblNcRecordMappingTable[`${value[0][i]}`]; + + syncLog(`NC API: dbTableRow.nestedAdd ${srcRow[1]}/hm/${dstRow[0]}/${dstRow[1]}`) + + await api.dbTableRow.nestedAdd( + 'noco', + syncDB.projectName, + table.title, + `${srcRow[1]}`, + 'mm', // fix me + `${field}`, + `${dstRow[1]}` + ); + } + } + })().catch(e => { + syncLog(`Link error`) + }); +} + +// fix me: +// instead of skipping data after retrieval, use select fields option in airtable API +function nocoBaseDataProcessing(table, record) { + (async () => { + let rec = record.fields; + + // kludge - + // trim spaces on either side of column name + // leads to error in NocoDB + Object.keys(rec).forEach(key => { + let replacedKey = key.replace(/\?/g, 'QQ').trim() + if (key !== replacedKey) { + rec[replacedKey] = rec[key]; + delete rec[key]; + } + }); + + // post-processing on the record + for (const [key, value] of Object.entries(rec) as any) { + // retrieve datatype + let dt = table.columns.find(x => x.title === key)?.uidt; + + // if(dt === undefined) + // console.log('fix me') + + // https://www.npmjs.com/package/validator + // default value: digits_after_decimal: [2] + // if currency, set decimal place to 2 + // + if (dt === 'Currency') rec[key] = value.toFixed(2); + + // we will pick up LTAR once all table data's are in place + if (dt === 'LinkToAnotherRecord') { + aTblDataLinks.push(JSON.parse(JSON.stringify(rec))); + delete rec[key]; + } + + // these will be automatically populated depending on schema configuration + if (dt === 'Lookup') delete rec[key]; + if (dt === 'Rollup') delete rec[key]; + + if (dt === 'Attachment') { + let tempArr = []; + for (const v of value) { + const binaryImage = await axios + .get(v.url, { + responseType: 'stream', + headers: { + 'Content-Type': v.type + } + }) + .then(response => { + return response.data; + }) + .catch(error => { + console.log(error); + return false; + }); + + var imageFile = new FormData(); + imageFile.append('files', binaryImage, { + filename: v.filename + }); + + const rs = await axios + .post(syncDB.baseURL + '/api/v1/db/storage/upload', imageFile, { + params: { + path: `noco/${syncDB.projectName}/${table.title}/${key}` + }, + headers: { + 'Content-Type': `multipart/form-data; boundary=${imageFile.getBoundary()}`, + 'xc-auth': syncDB.authToken + } + }) + .then(response => { + return response.data; + }) + .catch(e => { + console.log(e); + }); + + tempArr.push(...rs); + } + rec[key] = JSON.stringify(tempArr); + // rec[key] = JSON.stringify(tempArr); + } + } + + // insert airtable record ID explicitly into each records + // rec['record_id'] = record.id; + + // console.log(rec) + + syncLog(`dbTableRow.bulkCreate ${table.title} [${JSON.stringify(rec)}]`) + // console.log(JSON.stringify(rec, null, 2)) + + // bulk Insert + let returnValue = await api.dbTableRow.bulkCreate( + 'nc', + syncDB.projectName, + table.title, + [rec] + ); + + aTblNcRecordMappingTable[record.id] = [table.title, returnValue[0]]; + })().catch(e => { + syncLog(`Record insert error: ${e}`) + }); +} + +async function nocoReadData(table, callback) { + return new Promise((resolve, reject) => { + base(table.title) + .select({ + pageSize: 25, + // maxRecords: 1, + }) + .eachPage( + function page(records, fetchNextPage) { + // console.log(JSON.stringify(records, null, 2)); + + // This function (`page`) will get called for each page of records. + records.forEach(record => callback(table, record)); + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + function done(err) { + if (err) { + console.error(err); + reject(err) + } + resolve(true) + } + ); + }) +} + + +async function nocoReadDataSelected(table, callback, fields) { + return new Promise((resolve, reject) => { + + base(table.title) + .select({ + pageSize: 25, + // maxRecords: 100, + fields: [fields] + }) + .eachPage( + function page(records, fetchNextPage) { + // console.log(JSON.stringify(records, null, 2)); + + // This function (`page`) will get called for each page of records. + // records.forEach(record => callback(table, record)); + for (let i = 0; i < records.length; i++) { + callback(table, records[i], fields) + } + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + function done(err) { + if (err) { + console.error(err); + reject(err) + } + resolve(true) + } + ); + }); +} + +////////// +var global_ncCreatedProjectSchema: any = []; +var global_ncLinkMappingTable: any = []; + +function nc_isLinkExists(atblFieldId) { + if ( + global_ncLinkMappingTable.find( + x => x.aTbl.typeOptions.symmetricColumnId === atblFieldId + ) + ) + return true; + return false; +} + +// start function +async function nc_migrateATbl() { + + // fix me: delete project if already exists + // remove later + await init() + + // read schema file + const schema = getAtableSchema(); + let aTblSchema = schema.tableSchemas; + + // create empty project (XC-DB) + global_ncCreatedProjectSchema = await api.project.create({ + title: syncDB.projectName + }); + syncLog(`Create Project: ${syncDB.projectName}`) + + // prepare table schema (base) + await nocoCreateBaseSchema(schema); + + // add LTAR + await nocoCreateLinkToAnotherRecord(aTblSchema); + + // add look-ups + await nocoCreateLookups(aTblSchema); + + // add roll-ups + await nocoCreateRollups(aTblSchema); + + // configure primary values + await nocoSetPrimary(aTblSchema); + + // await nc_DumpTableSchema(); + let ncTblList = await api.dbTable.list(global_ncCreatedProjectSchema.id); + for (let i = 0; i < ncTblList.list.length; i++) { + let ncTbl = await api.dbTable.read(ncTblList.list[i].id); + await nocoReadData(ncTbl, nocoBaseDataProcessing); + } + + // // Configure link @ Data row's + for (let idx = 0; idx < global_ncLinkMappingTable.length; idx++) { + let x = global_ncLinkMappingTable[idx]; + let ncTbl = await nc_getTableSchema(aTbl_getTableName(x.aTbl.tblId).tn); + await nocoReadDataSelected(ncTbl, nocoLinkProcessing, x.aTbl.name); + } +} + +nc_migrateATbl().catch(e => { + console.log(e); +}); diff --git a/packages/nocodb/tests/sync/ts/tsconfig.json b/packages/nocodb/tests/sync/ts/tsconfig.json new file mode 100644 index 0000000000..c390c6e486 --- /dev/null +++ b/packages/nocodb/tests/sync/ts/tsconfig.json @@ -0,0 +1,63 @@ +{ + "compilerOptions": { + "skipLibCheck": false, + "composite": true, + "target": "es2017", + "outDir": "build", + "rootDir": "src", + "moduleResolution": "node", + "module": "commonjs", + "declaration": true, + "inlineSourceMap": true, + "esModuleInterop": true + /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, + "allowJs": false, + // "strict": true /* Enable all strict type-checking options. */, + + /* Strict Type-Checking Options */ + // "noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */, + // "strictNullChecks": true /* Enable strict null checks. */, + // "strictFunctionTypes": true /* Enable strict checking of function types. */, + // "strictPropertyInitialization": true /* Enable strict checking of property initialization in classes. */, + // "noImplicitThis": true /* Raise error on 'this' expressions with an implied 'any' type. */, + // "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */, + "resolveJsonModule": true, + /* Additional Checks */ + // "noUnusedLocals": true + // /* Report errors on unused locals. */, + // "noUnusedParameters": true + // /* Report errors on unused parameters. */, + "noImplicitReturns": true + /* Report error when not all code paths in function return a value. */, + "noFallthroughCasesInSwitch": true + /* Report errors for fallthrough cases in switch statement. */, + /* Debugging Options */ + "traceResolution": false + /* Report module resolution log messages. */, + "listEmittedFiles": false + /* Print names of generated files part of the compilation. */, + "listFiles": false + /* Print names of files part of the compilation. */, + "pretty": true + /* Stylize errors and messages using color and context. */, + /* Experimental Options */ + // "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */, + // "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */, + + "lib": [ + "es2017", + "dom" + ], + "types": [ + "node" + ], + "typeRoots": [ + "../../../node_modules/@types" + ] + }, + "include": [ + "src/**/*.ts", + "src/**/*.json" + ], + "compileOnSave": false +} \ No newline at end of file diff --git a/scripts/sdk/swagger.json b/scripts/sdk/swagger.json index 091505e9a9..b1ac9d4db8 100644 --- a/scripts/sdk/swagger.json +++ b/scripts/sdk/swagger.json @@ -1119,7 +1119,7 @@ "pinned": true, "deleted": true, "order": 0, - "column": [ + "columns": [ { "id": "string", "base_id": "string",