mirror of https://github.com/nocodb/nocodb
mertmit
3 years ago
committed by
Raju Udava
4 changed files with 1162 additions and 1 deletions
@ -0,0 +1,337 @@
|
||||
{ |
||||
"appBlanket": { |
||||
"userInfoById": { |
||||
"usrcTFn14vKTIgbW3": { |
||||
"id": "usrcTFn14vKTIgbW3", |
||||
"firstName": "Steyer", |
||||
"lastName": "Rom", |
||||
"email": "steyerrom@gmail.com", |
||||
"profilePicUrl": "https://static.airtable.com/images/userIcons/user_icon_9.png", |
||||
"permissionLevel": "owner", |
||||
"appBlanketUserState": "active" |
||||
} |
||||
}, |
||||
"externalAccountInfoById": {}, |
||||
"userGroupInfoById": {}, |
||||
"workspaceSyncSources": [], |
||||
"activeUserIdByAcceptedInviteId": {}, |
||||
"isWorkspaceOptedOutOfUserContentCdnAuth": false, |
||||
"isEnterpriseAccountOptedOutOfUserContentCdnAuth": false, |
||||
"enterpriseAttachmentRestrictions": { |
||||
"restrictionType": "unrestricted", |
||||
"attachmentTypeAllowlist": [] |
||||
}, |
||||
"isWorkspaceLinkedToEnterpriseAccount": false |
||||
}, |
||||
"description": null, |
||||
"sortTiebreakerKey": "appEHTLsc4lSaia9A", |
||||
"defaultViewMutability": null, |
||||
"maintenanceModeSettings": null, |
||||
"sharesById": { |
||||
"shrqM5QS9sSZ94mQx": { |
||||
"id": "shrqM5QS9sSZ94mQx", |
||||
"modelId": "appEHTLsc4lSaia9A", |
||||
"createdByUserId": "usrcTFn14vKTIgbW3", |
||||
"canBeCloned": false, |
||||
"canBeExported": false, |
||||
"includeHiddenColumns": false, |
||||
"includeBlocks": true, |
||||
"emailDomain": null, |
||||
"hasPassword": false, |
||||
"generationNumber": 0, |
||||
"metadata": null |
||||
} |
||||
}, |
||||
"workflowSectionsById": {}, |
||||
"applicationTransactionNumber": 21, |
||||
"tableSchemas": [ |
||||
{ |
||||
"id": "tblXYuhMZ3hWZkBCa", |
||||
"name": "Table 1", |
||||
"primaryColumnId": "fldrhmH0EYnOXfnUA", |
||||
"columns": [ |
||||
{ |
||||
"id": "fldrhmH0EYnOXfnUA", |
||||
"name": "Name", |
||||
"type": "text" |
||||
}, |
||||
{ |
||||
"id": "fldVFoK7t6aC92xzj", |
||||
"name": "Notes", |
||||
"type": "multilineText" |
||||
}, |
||||
{ |
||||
"id": "fld66bK6Pq8AG4m3h", |
||||
"name": "Attachments", |
||||
"type": "multipleAttachment", |
||||
"typeOptions": { |
||||
"unreversed": true |
||||
} |
||||
}, |
||||
{ |
||||
"id": "fldRC8zKoyGzM6agG", |
||||
"name": "Status", |
||||
"type": "select", |
||||
"typeOptions": { |
||||
"choices": { |
||||
"selQSYarqhTyVrwZw": { |
||||
"id": "selQSYarqhTyVrwZw", |
||||
"name": "Todo", |
||||
"color": "red" |
||||
}, |
||||
"selOFyoifOyV50QI7": { |
||||
"id": "selOFyoifOyV50QI7", |
||||
"name": "In progress", |
||||
"color": "yellow" |
||||
}, |
||||
"selsxseijq4XvcTB8": { |
||||
"id": "selsxseijq4XvcTB8", |
||||
"name": "Done", |
||||
"color": "green" |
||||
} |
||||
}, |
||||
"choiceOrder": [ |
||||
"selQSYarqhTyVrwZw", |
||||
"selOFyoifOyV50QI7", |
||||
"selsxseijq4XvcTB8" |
||||
] |
||||
} |
||||
} |
||||
], |
||||
"meaningfulColumnOrder": [ |
||||
{ |
||||
"columnId": "fldrhmH0EYnOXfnUA", |
||||
"visibility": true |
||||
}, |
||||
{ |
||||
"columnId": "fldVFoK7t6aC92xzj", |
||||
"visibility": true |
||||
}, |
||||
{ |
||||
"columnId": "fld66bK6Pq8AG4m3h", |
||||
"visibility": true |
||||
}, |
||||
{ |
||||
"columnId": "fldRC8zKoyGzM6agG", |
||||
"visibility": true |
||||
} |
||||
], |
||||
"views": [ |
||||
{ |
||||
"id": "viwbgKWGvUoZCosF1", |
||||
"name": "Grid view", |
||||
"type": "grid", |
||||
"createdByUserId": "usrcTFn14vKTIgbW3" |
||||
} |
||||
], |
||||
"viewOrder": [ |
||||
"viwbgKWGvUoZCosF1" |
||||
], |
||||
"viewsById": { |
||||
"viwbgKWGvUoZCosF1": { |
||||
"id": "viwbgKWGvUoZCosF1", |
||||
"name": "Grid view", |
||||
"type": "grid", |
||||
"createdByUserId": "usrcTFn14vKTIgbW3" |
||||
} |
||||
}, |
||||
"viewSectionsById": {}, |
||||
"schemaChecksum": "412180368d81674e723b957501f16a57c9264fc69d19668b70d1547888c29413" |
||||
} |
||||
], |
||||
"tableDatas": [ |
||||
{ |
||||
"id": "tblXYuhMZ3hWZkBCa", |
||||
"rows": [ |
||||
{ |
||||
"id": "recv9Z8uFFNt50rqX", |
||||
"createdTime": "2022-04-27T18:48:37.000Z", |
||||
"cellValuesByColumnId": { |
||||
"fld66bK6Pq8AG4m3h": [ |
||||
{ |
||||
"id": "attdRIU80oCDC8u6X", |
||||
"url": "https://dl.airtable.com/.attachments/247a5881e7742c2d55cb8f814fe7263a/964d1018/512x512.png", |
||||
"filename": "512x512.png", |
||||
"type": "image/png", |
||||
"size": 77822, |
||||
"width": 2763, |
||||
"height": 2763, |
||||
"smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/f5f0717d00f1f61c402fec203d16efd7/61bea79e", |
||||
"smallThumbWidth": 36, |
||||
"smallThumbHeight": 36, |
||||
"largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/8f49ea3dcdaf3aa9788e84f1a3e3f3e2/81284101", |
||||
"largeThumbWidth": 512, |
||||
"largeThumbHeight": 512, |
||||
"fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/c5300e8cda92de966de82c760cd44533/89d51367", |
||||
"fullThumbWidth": 3000, |
||||
"fullThumbHeight": 3000 |
||||
} |
||||
], |
||||
"fldrhmH0EYnOXfnUA": "nc" |
||||
} |
||||
}, |
||||
{ |
||||
"id": "recmAktd3OQe3Wg8C", |
||||
"createdTime": "2022-04-27T18:48:37.000Z", |
||||
"cellValuesByColumnId": { |
||||
"fld66bK6Pq8AG4m3h": [ |
||||
{ |
||||
"id": "attLBB2eqE9grLlUU", |
||||
"url": "https://dl.airtable.com/.attachments/a67aaa1efa29d40c3633ca03f0b366e6/9b56c6e2/Abstract-Nord.png", |
||||
"filename": "Abstract-Nord.png", |
||||
"type": "image/png", |
||||
"size": 140219, |
||||
"width": 1920, |
||||
"height": 1080, |
||||
"smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/d5daa7de864a8302ad9d2c43a257a52d/2b709aa8", |
||||
"smallThumbWidth": 64, |
||||
"smallThumbHeight": 36, |
||||
"largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/7b388a8484a1bc8f9f91bbf304c9b42e/4663133b", |
||||
"largeThumbWidth": 910, |
||||
"largeThumbHeight": 512, |
||||
"fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/4d5dfa21efa34198c740443551531eea/ea51b594", |
||||
"fullThumbWidth": 3000, |
||||
"fullThumbHeight": 3000 |
||||
}, |
||||
{ |
||||
"id": "attIzZpRBBWwRI5Wz", |
||||
"url": "https://dl.airtable.com/.attachments/107f2f1d886b1f1ae0d528ba7f76df03/931109ad/archlinux.png", |
||||
"filename": "archlinux.png", |
||||
"type": "image/png", |
||||
"size": 184887, |
||||
"width": 3840, |
||||
"height": 2160, |
||||
"smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/cb9894ae30f7a264a87639d5ce4980ec/e000fb24", |
||||
"smallThumbWidth": 64, |
||||
"smallThumbHeight": 36, |
||||
"largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/1f4146a90346325a8ab51d5d7f4f40be/9a73928f", |
||||
"largeThumbWidth": 910, |
||||
"largeThumbHeight": 512, |
||||
"fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/9954669a1ba7bc2d4e4d50f9c0130435/7d46d64a", |
||||
"fullThumbWidth": 3000, |
||||
"fullThumbHeight": 3000 |
||||
}, |
||||
{ |
||||
"id": "atttj7OPAZ1iDtHBt", |
||||
"url": "https://dl.airtable.com/.attachments/4eec95e15a951829c2b12e4375bece7f/8ff26ede/arctic-landscape.png", |
||||
"filename": "arctic-landscape.png", |
||||
"type": "image/png", |
||||
"size": 155548, |
||||
"width": 1920, |
||||
"height": 1080, |
||||
"smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/dd63a0d18ed1c4bf3e7096145fc5fa0f/ebfc95ac", |
||||
"smallThumbWidth": 64, |
||||
"smallThumbHeight": 36, |
||||
"largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/c9b441a88304b53cdc44fa0fcfdb57e6/8c6b629d", |
||||
"largeThumbWidth": 910, |
||||
"largeThumbHeight": 512, |
||||
"fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/f5b74f24a31f22e8e859954499abfa67/fd11b916", |
||||
"fullThumbWidth": 3000, |
||||
"fullThumbHeight": 3000 |
||||
} |
||||
], |
||||
"fldrhmH0EYnOXfnUA": "wp" |
||||
} |
||||
}, |
||||
{ |
||||
"id": "recAFj2eQVsynoDJa", |
||||
"createdTime": "2022-04-27T18:48:37.000Z", |
||||
"cellValuesByColumnId": { |
||||
"fldrhmH0EYnOXfnUA": "test" |
||||
} |
||||
} |
||||
], |
||||
"viewDatas": [ |
||||
{ |
||||
"id": "viwbgKWGvUoZCosF1", |
||||
"frozenColumnCount": 1, |
||||
"columnOrder": [ |
||||
{ |
||||
"columnId": "fldrhmH0EYnOXfnUA", |
||||
"visibility": true |
||||
}, |
||||
{ |
||||
"columnId": "fldVFoK7t6aC92xzj", |
||||
"visibility": true |
||||
}, |
||||
{ |
||||
"columnId": "fld66bK6Pq8AG4m3h", |
||||
"visibility": true |
||||
}, |
||||
{ |
||||
"columnId": "fldRC8zKoyGzM6agG", |
||||
"visibility": true |
||||
} |
||||
], |
||||
"sharesById": {}, |
||||
"createdByUserId": "usrcTFn14vKTIgbW3", |
||||
"applicationTransactionNumber": 21, |
||||
"rowOrder": [ |
||||
{ |
||||
"rowId": "recv9Z8uFFNt50rqX", |
||||
"visibility": true |
||||
}, |
||||
{ |
||||
"rowId": "recmAktd3OQe3Wg8C", |
||||
"visibility": true |
||||
}, |
||||
{ |
||||
"rowId": "recAFj2eQVsynoDJa", |
||||
"visibility": true |
||||
} |
||||
] |
||||
} |
||||
], |
||||
"hasOnlyIncludedRowAndCellDataForIncludedViews": false |
||||
} |
||||
], |
||||
"hasBlockInstallations": false, |
||||
"applicationAdminFlags": { |
||||
"UPDATE_PRIMITIVE_CELL_THROTTLE_MS": null, |
||||
"MAX_WORKFLOWS_PER_APPLICATION": null, |
||||
"MAX_SYNC_SOURCES_PER_APPLICATION": null, |
||||
"MAX_SYNC_SOURCES_PER_TABLE": null, |
||||
"MAX_SYNCED_TABLES_PER_APPLICATION": null, |
||||
"CUSTOM_MAX_NUM_ROWS_PER_TABLE": null |
||||
}, |
||||
"pageBundles": [], |
||||
"uploadedUserContentCdnSetting": { |
||||
"applicationScopedAuthMode": "public" |
||||
}, |
||||
"applicationV2TargetedFeatureFlagClientConfiguration": { |
||||
"nonCollaboratorsInCollaboratorField": { |
||||
"trafficLevel": 0 |
||||
}, |
||||
"applicationInsights": { |
||||
"trafficLevel": 0 |
||||
}, |
||||
"autoOpenInsightsPaneOnUnseenSuggestion": { |
||||
"trafficLevel": 0 |
||||
}, |
||||
"disabledWorkflowOnSchemaChangeSuggestion": { |
||||
"trafficLevel": 0 |
||||
}, |
||||
"syncFailureSuggestion": { |
||||
"trafficLevel": 0 |
||||
}, |
||||
"unusedViewsSuggestion": { |
||||
"trafficLevel": 0 |
||||
}, |
||||
"filterUnusedViewsUsingDependencyGraph": { |
||||
"trafficLevel": 100 |
||||
}, |
||||
"unusedSelectChoicesSuggestion": { |
||||
"trafficLevel": 0 |
||||
}, |
||||
"unifiedEventLog": { |
||||
"trafficLevel": 0 |
||||
}, |
||||
"constantPoolingForCrudResponses": { |
||||
"trafficLevel": 0 |
||||
} |
||||
}, |
||||
"applicationV2EnabledFeatureNames": [ |
||||
"filterUnusedViewsUsingDependencyGraph" |
||||
], |
||||
"isConstantPooledData": false |
||||
} |
||||
@ -0,0 +1,761 @@
|
||||
import { Api, UITypes } from 'nocodb-sdk'; |
||||
import Airtable from 'airtable'; |
||||
import jsonfile from 'jsonfile'; |
||||
import FormData from 'form-data' |
||||
import axios from 'axios'; |
||||
|
||||
|
||||
//RUN: npx ts-node src/nc-sync.ts
|
||||
|
||||
function syncLog(log) { |
||||
console.log(`nc-sync: ${log}`) |
||||
} |
||||
|
||||
// apiKey & baseID configurations required to read data using Airtable APIs
|
||||
//
|
||||
const syncDB = { |
||||
airtable: { |
||||
apiKey: 'keyfaOQmPOpigyJV8', |
||||
baseId: 'appEHTLsc4lSaia9A', |
||||
schemaJson: 'attachment.json' |
||||
}, |
||||
projectName: 'sample', |
||||
baseURL: 'http://localhost:8080', |
||||
authToken: |
||||
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InRlc3RAdGVzdC50ZXN0IiwiZmlyc3RuYW1lIjpudWxsLCJsYXN0bmFtZSI6bnVsbCwiaWQiOiJ1c183eHJ5b25jYWZzbHd2diIsInJvbGVzIjoidXNlcixzdXBlciIsImlhdCI6MTY1MDcxMjMyN30.zB_E46qkQy1mCqjjJL89WPa1jCY101BAAoLAyE7b1n8' |
||||
}; |
||||
|
||||
const api = new Api({ |
||||
baseURL: syncDB.baseURL, |
||||
headers: { |
||||
'xc-auth': syncDB.authToken |
||||
} |
||||
}); |
||||
|
||||
// global schema store
|
||||
let global_schema: any = getAtableSchema().tableSchemas; |
||||
|
||||
function getAtableSchema() { |
||||
return jsonfile.readFileSync(syncDB.airtable.schemaJson); |
||||
} |
||||
|
||||
// base mapping table
|
||||
const aTblNcTypeMap = { |
||||
foreignKey: UITypes.LinkToAnotherRecord, |
||||
text: UITypes.SingleLineText, |
||||
multilineText: UITypes.LongText, |
||||
multipleAttachment: UITypes.Attachment, |
||||
checkbox: UITypes.Checkbox, |
||||
multiSelect: UITypes.MultiSelect, |
||||
select: UITypes.SingleSelect, |
||||
collaborator: UITypes.Collaborator, |
||||
date: UITypes.Date, |
||||
// kludge: phone: UITypes.PhoneNumber,
|
||||
phone: UITypes.SingleLineText, |
||||
number: UITypes.Number, |
||||
rating: UITypes.Rating, |
||||
// kludge: formula: UITypes.Formula,
|
||||
formula: UITypes.SingleLineText, |
||||
rollup: UITypes.Rollup, |
||||
count: UITypes.Count, |
||||
lookup: UITypes.Lookup, |
||||
autoNumber: UITypes.AutoNumber, |
||||
barcode: UITypes.Barcode, |
||||
button: UITypes.Button |
||||
}; |
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// aTbl helper routines
|
||||
//
|
||||
|
||||
// aTbl: retrieve table name from table ID
|
||||
//
|
||||
function aTbl_getTableName(tblId) { |
||||
const sheetObj = global_schema.find(tbl => tbl.id === tblId); |
||||
return { |
||||
tn: sheetObj.name |
||||
}; |
||||
} |
||||
|
||||
// aTbl: retrieve column name from column ID
|
||||
//
|
||||
function aTbl_getColumnName(colId) { |
||||
for (let i = 0; i < global_schema.length; i++) { |
||||
let sheetObj = global_schema[i]; |
||||
const column = sheetObj.columns.find(col => col.id === colId); |
||||
if (column !== undefined) |
||||
return { |
||||
tn: sheetObj.name, |
||||
cn: column.name |
||||
}; |
||||
} |
||||
return {}; |
||||
} |
||||
|
||||
// nc dump schema
|
||||
/* |
||||
async function nc_DumpTableSchema() { |
||||
console.log('['); |
||||
let ncTblList = await api.dbTable.list(global_ncCreatedProjectSchema.id); |
||||
for (let i = 0; i < ncTblList.list.length; i++) { |
||||
let ncTbl = await api.dbTable.read(ncTblList.list[i].id); |
||||
console.log(JSON.stringify(ncTbl, null, 2)); |
||||
console.log(','); |
||||
} |
||||
console.log(']'); |
||||
} |
||||
*/ |
||||
|
||||
// retrieve nc column schema from using aTbl field ID as reference
|
||||
//
|
||||
async function nc_getColumnSchema(aTblFieldId) { |
||||
let ncTblList = await api.dbTable.list(global_ncCreatedProjectSchema.id); |
||||
let aTblField = aTbl_getColumnName(aTblFieldId); |
||||
let ncTblId = ncTblList.list.filter(x => x.title === aTblField.tn)[0].id; |
||||
let ncTbl = await api.dbTable.read(ncTblId); |
||||
let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn); |
||||
return ncCol; |
||||
} |
||||
|
||||
// retrieve nc table schema using table name
|
||||
async function nc_getTableSchema(tableName) { |
||||
let ncTblList = await api.dbTable.list(global_ncCreatedProjectSchema.id); |
||||
let ncTblId = ncTblList.list.filter(x => x.title === tableName)[0].id; |
||||
let ncTbl = await api.dbTable.read(ncTblId); |
||||
return ncTbl; |
||||
} |
||||
|
||||
// delete project if already exists
|
||||
async function init() { |
||||
// delete 'sample' project if already exists
|
||||
let x = await api.project.list() |
||||
|
||||
let sampleProj = x.list.find(a => a.title === syncDB.projectName) |
||||
if (sampleProj) { |
||||
await api.project.delete(sampleProj.id) |
||||
} |
||||
|
||||
syncLog('Init') |
||||
} |
||||
|
||||
// map UIDT
|
||||
//
|
||||
function getNocoType(col) { |
||||
// start with default map
|
||||
let ncType = aTblNcTypeMap[col.type]; |
||||
|
||||
// types email & url are marked as text
|
||||
// types currency & percent, duration are marked as number
|
||||
// types createTime & modifiedTime are marked as formula
|
||||
|
||||
switch (col.type) { |
||||
case 'text': |
||||
if (col.typeOptions?.validatorName === 'email') ncType = UITypes.Email; |
||||
else if (col.typeOptions?.validatorName === 'url') ncType = UITypes.URL; |
||||
break; |
||||
|
||||
case 'number': |
||||
// kludge: currency validation error with decimal places
|
||||
if (col.typeOptions?.format === 'percentV2') ncType = UITypes.Percent; |
||||
else if (col.typeOptions?.format === 'duration') |
||||
ncType = UITypes.Duration; |
||||
else if (col.typeOptions?.format === 'currency') |
||||
ncType = UITypes.Currency; |
||||
break; |
||||
|
||||
case 'formula': |
||||
if (col.typeOptions?.formulaTextParsed === 'CREATED_TIME()') |
||||
ncType = UITypes.CreateTime; |
||||
else if (col.typeOptions?.formulaTextParsed === 'LAST_MODIFIED_TIME()') |
||||
ncType = UITypes.LastModifiedTime; |
||||
break; |
||||
} |
||||
|
||||
return ncType; |
||||
} |
||||
|
||||
// retrieve additional options associated with selected data types
|
||||
//
|
||||
function getNocoTypeOptions(col) { |
||||
switch (col.type) { |
||||
case 'select': |
||||
case 'multiSelect': |
||||
// prepare options list in CSV format
|
||||
// note: NC doesn't allow comma's in options
|
||||
//
|
||||
let opt = []; |
||||
for (let [, value] of Object.entries(col.typeOptions.choices) as any) { |
||||
opt.push(value.name); |
||||
} |
||||
let csvOpt = "'" + opt.join("','") + "'"; |
||||
return { type: 'select', data: csvOpt }; |
||||
|
||||
default: |
||||
return { type: undefined }; |
||||
} |
||||
} |
||||
|
||||
// convert to Nc schema (basic, excluding relations)
|
||||
//
|
||||
function tablesPrepare(tblSchema) { |
||||
let tables = []; |
||||
for (let i = 0; i < tblSchema.length; ++i) { |
||||
let table: any = {}; |
||||
|
||||
syncLog(`Preparing base schema (sans relations): ${tblSchema[i].name}`) |
||||
|
||||
// table name
|
||||
table.table_name = tblSchema[i].name; |
||||
table.title = tblSchema[i].name; |
||||
|
||||
// insert record_id of type ID by default
|
||||
table.columns = [ |
||||
{ |
||||
title: 'record_id', |
||||
column_name: 'record_id', |
||||
uidt: UITypes.ID |
||||
// uidt: UITypes.SingleLineText,
|
||||
// pk: true
|
||||
} |
||||
]; |
||||
|
||||
for (let j = 0; j < tblSchema[i].columns.length; j++) { |
||||
let col = tblSchema[i].columns[j]; |
||||
|
||||
// skip link, lookup, rollup fields in this iteration
|
||||
if (['foreignKey', 'lookup', 'rollup'].includes(col.type)) continue; |
||||
|
||||
// not supported datatype
|
||||
// if (['formula'].includes(col.type)) continue;
|
||||
|
||||
// base column schema
|
||||
// kludge: error observed in Nc with space around column-name
|
||||
let ncCol: any = { |
||||
title: col.name.trim(), |
||||
|
||||
// knex complains use of '?' in field name
|
||||
//column_name: col.name.replace(/\?/g, '\\?').trim(),
|
||||
column_name: col.name.replace(/\?/g, 'QQ').trim(), |
||||
uidt: getNocoType(col) |
||||
}; |
||||
|
||||
// additional column parameters when applicable
|
||||
let colOptions = getNocoTypeOptions(col); |
||||
|
||||
switch (colOptions.type) { |
||||
case 'select': |
||||
ncCol.dtxp = colOptions.data; |
||||
break; |
||||
|
||||
case undefined: |
||||
break; |
||||
} |
||||
|
||||
table.columns.push(ncCol); |
||||
} |
||||
|
||||
tables.push(table); |
||||
} |
||||
return tables; |
||||
} |
||||
|
||||
async function nocoCreateBaseSchema(srcSchema) { |
||||
// base schema preparation: exclude
|
||||
let tables = tablesPrepare(srcSchema.tableSchemas); |
||||
|
||||
// for each table schema, create nc table
|
||||
for (let idx = 0; idx < tables.length; idx++) { |
||||
|
||||
syncLog(`dbTable.create ${tables[idx].title}`) |
||||
let table = await api.dbTable.create( |
||||
global_ncCreatedProjectSchema.id, |
||||
tables[idx] |
||||
); |
||||
} |
||||
|
||||
// debug
|
||||
// console.log(JSON.stringify(tables, null, 2));
|
||||
return tables; |
||||
} |
||||
|
||||
async function nocoCreateLinkToAnotherRecord(aTblSchema) { |
||||
// Link to another RECORD
|
||||
for (let idx = 0; idx < aTblSchema.length; idx++) { |
||||
let aTblLinkColumns = aTblSchema[idx].columns.filter( |
||||
x => x.type === 'foreignKey' |
||||
); |
||||
|
||||
// Link columns exist
|
||||
//
|
||||
if (aTblLinkColumns.length) { |
||||
for (let i = 0; i < aTblLinkColumns.length; i++) { |
||||
|
||||
{ |
||||
let src = aTbl_getColumnName(aTblLinkColumns[i].id) |
||||
let dst = aTbl_getColumnName(aTblLinkColumns[i].typeOptions.symmetricColumnId) |
||||
syncLog(` LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst.tn}:${dst.cn}`) |
||||
} |
||||
|
||||
// check if link already established?
|
||||
if (!nc_isLinkExists(aTblLinkColumns[i].id)) { |
||||
// parent table ID
|
||||
let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; |
||||
|
||||
// find child table name from symmetric column ID specified
|
||||
let childTable = aTbl_getColumnName( |
||||
aTblLinkColumns[i].typeOptions.symmetricColumnId |
||||
); |
||||
|
||||
// retrieve child table ID (nc) from table name
|
||||
let childTableId = (await nc_getTableSchema(childTable.tn)).id; |
||||
|
||||
// create link
|
||||
let column = await api.dbTableColumn.create(srcTableId, { |
||||
uidt: 'LinkToAnotherRecord', |
||||
title: aTblLinkColumns[i].name, |
||||
parentId: srcTableId, |
||||
childId: childTableId, |
||||
type: 'mm' |
||||
// aTblLinkColumns[i].typeOptions.relationship === 'many'
|
||||
// ? 'mm'
|
||||
// : 'hm'
|
||||
}); |
||||
syncLog(`NC API: dbTableColumn.create LinkToAnotherRecord`) |
||||
|
||||
// store link information in separate table
|
||||
// this information will be helpful in identifying relation pair
|
||||
let link = { |
||||
nc: { |
||||
title: aTblLinkColumns[i].name, |
||||
parentId: srcTableId, |
||||
childId: childTableId, |
||||
type: 'mm' |
||||
}, |
||||
aTbl: { |
||||
tblId: aTblSchema[idx].id, |
||||
...aTblLinkColumns[i] |
||||
} |
||||
}; |
||||
|
||||
global_ncLinkMappingTable.push(link); |
||||
} else { |
||||
// if link already exists, we need to change name of linked column
|
||||
// to what is represented in airtable
|
||||
|
||||
// 1. extract associated link information from link table
|
||||
// 2. retrieve parent table information (source)
|
||||
// 3. using foreign parent & child column ID, find associated mapping in child table
|
||||
// 4. update column name
|
||||
let x = global_ncLinkMappingTable.findIndex( |
||||
x => |
||||
x.aTbl.tblId === aTblLinkColumns[i].typeOptions.foreignTableId && |
||||
x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId |
||||
); |
||||
|
||||
let childTblSchema = await api.dbTable.read( |
||||
global_ncLinkMappingTable[x].nc.childId |
||||
); |
||||
let parentTblSchema = await api.dbTable.read( |
||||
global_ncLinkMappingTable[x].nc.parentId |
||||
); |
||||
|
||||
let parentLinkColumn: any = parentTblSchema.columns.find( |
||||
col => col.title === global_ncLinkMappingTable[x].nc.title |
||||
); |
||||
|
||||
let childLinkColumn: any = {}; |
||||
if (parentLinkColumn.colOptions.type == 'hm') { |
||||
// for hm:
|
||||
// mapping between child & parent column id is direct
|
||||
//
|
||||
childLinkColumn = childTblSchema.columns.find( |
||||
(col: any) => |
||||
col.uidt === 'LinkToAnotherRecord' && |
||||
col.colOptions.fk_child_column_id === |
||||
parentLinkColumn.colOptions.fk_child_column_id && |
||||
col.colOptions.fk_parent_column_id === |
||||
parentLinkColumn.colOptions.fk_parent_column_id |
||||
); |
||||
} else { |
||||
// for mm:
|
||||
// mapping between child & parent column id is inverted
|
||||
//
|
||||
childLinkColumn = childTblSchema.columns.find( |
||||
(col: any) => |
||||
col.uidt === 'LinkToAnotherRecord' && |
||||
col.colOptions.fk_child_column_id === |
||||
parentLinkColumn.colOptions.fk_parent_column_id && |
||||
col.colOptions.fk_parent_column_id === |
||||
parentLinkColumn.colOptions.fk_child_column_id && |
||||
col.colOptions.fk_mm_model_id === |
||||
parentLinkColumn.colOptions.fk_mm_model_id |
||||
); |
||||
} |
||||
|
||||
// rename
|
||||
// note that: current rename API requires us to send all parameters,
|
||||
// not just title being renamed
|
||||
let res = await api.dbTableColumn.update(childLinkColumn.id, { |
||||
...childLinkColumn, |
||||
title: aTblLinkColumns[i].name, |
||||
}); |
||||
// console.log(res.columns.find(x => x.title === aTblLinkColumns[i].name))
|
||||
syncLog(`dbTableColumn.update rename symmetric column`) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
async function nocoCreateLookups(aTblSchema) { |
||||
// LookUps
|
||||
for (let idx = 0; idx < aTblSchema.length; idx++) { |
||||
let aTblColumns = aTblSchema[idx].columns.filter(x => x.type === 'lookup'); |
||||
|
||||
// parent table ID
|
||||
let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; |
||||
|
||||
if (aTblColumns.length) { |
||||
// Lookup
|
||||
for (let i = 0; i < aTblColumns.length; i++) { |
||||
let ncRelationColumn = await nc_getColumnSchema( |
||||
aTblColumns[i].typeOptions.relationColumnId |
||||
); |
||||
let ncLookupColumn = await nc_getColumnSchema( |
||||
aTblColumns[i].typeOptions.foreignTableRollupColumnId |
||||
); |
||||
|
||||
let lookupColumn = await api.dbTableColumn.create(srcTableId, { |
||||
uidt: 'Lookup', |
||||
title: aTblColumns[i].name, |
||||
fk_relation_column_id: ncRelationColumn.id, |
||||
fk_lookup_column_id: ncLookupColumn.id |
||||
}); |
||||
|
||||
syncLog(`NC API: dbTableColumn.create LOOKUP`) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
async function nocoCreateRollups(aTblSchema) { |
||||
// Rollups
|
||||
for (let idx = 0; idx < aTblSchema.length; idx++) { |
||||
let aTblColumns = aTblSchema[idx].columns.filter(x => x.type === 'rollup'); |
||||
|
||||
// parent table ID
|
||||
let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; |
||||
|
||||
if (aTblColumns.length) { |
||||
// rollup exist
|
||||
for (let i = 0; i < aTblColumns.length; i++) { |
||||
let ncRelationColumn = await nc_getColumnSchema( |
||||
aTblColumns[i].typeOptions.relationColumnId |
||||
); |
||||
let ncRollupColumn = await nc_getColumnSchema( |
||||
aTblColumns[i].typeOptions.foreignTableRollupColumnId |
||||
); |
||||
|
||||
let lookupColumn = await api.dbTableColumn.create(srcTableId, { |
||||
uidt: 'Rollup', |
||||
title: aTblColumns[i].name, |
||||
fk_relation_column_id: ncRelationColumn.id, |
||||
fk_rollup_column_id: ncRollupColumn.id, |
||||
rollup_function: 'sum' // fix me: hardwired
|
||||
}); |
||||
|
||||
syncLog(`NC API: dbTableColumn.create ROLLUP`) |
||||
|
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
async function nocoSetPrimary(aTblSchema) { |
||||
for (let idx = 0; idx < aTblSchema.length; idx++) { |
||||
let pColId = aTblSchema[idx].primaryColumnId; |
||||
let ncCol = await nc_getColumnSchema(pColId); |
||||
|
||||
syncLog(`NC API: dbTableColumn.primaryColumnSet`) |
||||
await api.dbTableColumn.primaryColumnSet(ncCol.id); |
||||
} |
||||
} |
||||
|
||||
////////// Data processing
|
||||
|
||||
// https://www.airtable.com/app1ivUy7ba82jOPn/api/docs#javascript/metadata
|
||||
let base = new Airtable({ apiKey: syncDB.airtable.apiKey }).base( |
||||
syncDB.airtable.baseId |
||||
); |
||||
|
||||
let aTblDataLinks = []; |
||||
let aTblNcRecordMappingTable = {}; |
||||
|
||||
function nocoLinkProcessing(table, record, field) { |
||||
(async () => { |
||||
let rec = record.fields; |
||||
const value = Object.values(rec) as any; |
||||
let srcRow = aTblNcRecordMappingTable[`${record.id}`]; |
||||
|
||||
if (value.length) { |
||||
for (let i = 0; i < value[0].length; i++) { |
||||
let dstRow = aTblNcRecordMappingTable[`${value[0][i]}`]; |
||||
|
||||
syncLog(`NC API: dbTableRow.nestedAdd ${srcRow[1]}/hm/${dstRow[0]}/${dstRow[1]}`) |
||||
|
||||
await api.dbTableRow.nestedAdd( |
||||
'noco', |
||||
syncDB.projectName, |
||||
table.title, |
||||
`${srcRow[1]}`, |
||||
'mm', // fix me
|
||||
`${field}`, |
||||
`${dstRow[1]}` |
||||
); |
||||
} |
||||
} |
||||
})().catch(e => { |
||||
syncLog(`Link error`) |
||||
}); |
||||
} |
||||
|
||||
// fix me:
|
||||
// instead of skipping data after retrieval, use select fields option in airtable API
|
||||
function nocoBaseDataProcessing(table, record) { |
||||
(async () => { |
||||
let rec = record.fields; |
||||
|
||||
// kludge -
|
||||
// trim spaces on either side of column name
|
||||
// leads to error in NocoDB
|
||||
Object.keys(rec).forEach(key => { |
||||
let replacedKey = key.replace(/\?/g, 'QQ').trim() |
||||
if (key !== replacedKey) { |
||||
rec[replacedKey] = rec[key]; |
||||
delete rec[key]; |
||||
} |
||||
}); |
||||
|
||||
// post-processing on the record
|
||||
for (const [key, value] of Object.entries(rec) as any) { |
||||
// retrieve datatype
|
||||
let dt = table.columns.find(x => x.title === key)?.uidt; |
||||
|
||||
// if(dt === undefined)
|
||||
// console.log('fix me')
|
||||
|
||||
// https://www.npmjs.com/package/validator
|
||||
// default value: digits_after_decimal: [2]
|
||||
// if currency, set decimal place to 2
|
||||
//
|
||||
if (dt === 'Currency') rec[key] = value.toFixed(2); |
||||
|
||||
// we will pick up LTAR once all table data's are in place
|
||||
if (dt === 'LinkToAnotherRecord') { |
||||
aTblDataLinks.push(JSON.parse(JSON.stringify(rec))); |
||||
delete rec[key]; |
||||
} |
||||
|
||||
// these will be automatically populated depending on schema configuration
|
||||
if (dt === 'Lookup') delete rec[key]; |
||||
if (dt === 'Rollup') delete rec[key]; |
||||
|
||||
if (dt === 'Attachment') { |
||||
let tempArr = []; |
||||
for (const v of value) { |
||||
const binaryImage = await axios |
||||
.get(v.url, { |
||||
responseType: 'stream', |
||||
headers: { |
||||
'Content-Type': v.type |
||||
} |
||||
}) |
||||
.then(response => { |
||||
return response.data; |
||||
}) |
||||
.catch(error => { |
||||
console.log(error); |
||||
return false; |
||||
}); |
||||
|
||||
var imageFile = new FormData(); |
||||
imageFile.append('files', binaryImage, { |
||||
filename: v.filename |
||||
}); |
||||
|
||||
const rs = await axios |
||||
.post(syncDB.baseURL + '/api/v1/db/storage/upload', imageFile, { |
||||
params: { |
||||
path: `noco/${syncDB.projectName}/${table.title}/${key}` |
||||
}, |
||||
headers: { |
||||
'Content-Type': `multipart/form-data; boundary=${imageFile.getBoundary()}`, |
||||
'xc-auth': syncDB.authToken |
||||
} |
||||
}) |
||||
.then(response => { |
||||
return response.data; |
||||
}) |
||||
.catch(e => { |
||||
console.log(e); |
||||
}); |
||||
|
||||
tempArr.push(...rs); |
||||
} |
||||
rec[key] = JSON.stringify(tempArr); |
||||
// rec[key] = JSON.stringify(tempArr);
|
||||
} |
||||
} |
||||
|
||||
// insert airtable record ID explicitly into each records
|
||||
// rec['record_id'] = record.id;
|
||||
|
||||
// console.log(rec)
|
||||
|
||||
syncLog(`dbTableRow.bulkCreate ${table.title} [${JSON.stringify(rec)}]`) |
||||
// console.log(JSON.stringify(rec, null, 2))
|
||||
|
||||
// bulk Insert
|
||||
let returnValue = await api.dbTableRow.bulkCreate( |
||||
'nc', |
||||
syncDB.projectName, |
||||
table.title, |
||||
[rec] |
||||
); |
||||
|
||||
aTblNcRecordMappingTable[record.id] = [table.title, returnValue[0]]; |
||||
})().catch(e => { |
||||
syncLog(`Record insert error: ${e}`) |
||||
}); |
||||
} |
||||
|
||||
async function nocoReadData(table, callback) { |
||||
return new Promise((resolve, reject) => { |
||||
base(table.title) |
||||
.select({ |
||||
pageSize: 25, |
||||
// maxRecords: 1,
|
||||
}) |
||||
.eachPage( |
||||
function page(records, fetchNextPage) { |
||||
// console.log(JSON.stringify(records, null, 2));
|
||||
|
||||
// This function (`page`) will get called for each page of records.
|
||||
records.forEach(record => callback(table, record)); |
||||
|
||||
// To fetch the next page of records, call `fetchNextPage`.
|
||||
// If there are more records, `page` will get called again.
|
||||
// If there are no more records, `done` will get called.
|
||||
fetchNextPage(); |
||||
}, |
||||
function done(err) { |
||||
if (err) { |
||||
console.error(err); |
||||
reject(err) |
||||
} |
||||
resolve(true) |
||||
} |
||||
); |
||||
}) |
||||
} |
||||
|
||||
|
||||
async function nocoReadDataSelected(table, callback, fields) { |
||||
return new Promise((resolve, reject) => { |
||||
|
||||
base(table.title) |
||||
.select({ |
||||
pageSize: 25, |
||||
// maxRecords: 100,
|
||||
fields: [fields] |
||||
}) |
||||
.eachPage( |
||||
function page(records, fetchNextPage) { |
||||
// console.log(JSON.stringify(records, null, 2));
|
||||
|
||||
// This function (`page`) will get called for each page of records.
|
||||
// records.forEach(record => callback(table, record));
|
||||
for (let i = 0; i < records.length; i++) { |
||||
callback(table, records[i], fields) |
||||
} |
||||
|
||||
// To fetch the next page of records, call `fetchNextPage`.
|
||||
// If there are more records, `page` will get called again.
|
||||
// If there are no more records, `done` will get called.
|
||||
fetchNextPage(); |
||||
}, |
||||
function done(err) { |
||||
if (err) { |
||||
console.error(err); |
||||
reject(err) |
||||
} |
||||
resolve(true) |
||||
} |
||||
); |
||||
}); |
||||
} |
||||
|
||||
//////////
|
||||
var global_ncCreatedProjectSchema: any = []; |
||||
var global_ncLinkMappingTable: any = []; |
||||
|
||||
function nc_isLinkExists(atblFieldId) { |
||||
if ( |
||||
global_ncLinkMappingTable.find( |
||||
x => x.aTbl.typeOptions.symmetricColumnId === atblFieldId |
||||
) |
||||
) |
||||
return true; |
||||
return false; |
||||
} |
||||
|
||||
// start function
|
||||
async function nc_migrateATbl() { |
||||
|
||||
// fix me: delete project if already exists
|
||||
// remove later
|
||||
await init() |
||||
|
||||
// read schema file
|
||||
const schema = getAtableSchema(); |
||||
let aTblSchema = schema.tableSchemas; |
||||
|
||||
// create empty project (XC-DB)
|
||||
global_ncCreatedProjectSchema = await api.project.create({ |
||||
title: syncDB.projectName |
||||
}); |
||||
syncLog(`Create Project: ${syncDB.projectName}`) |
||||
|
||||
// prepare table schema (base)
|
||||
await nocoCreateBaseSchema(schema); |
||||
|
||||
// add LTAR
|
||||
await nocoCreateLinkToAnotherRecord(aTblSchema); |
||||
|
||||
// add look-ups
|
||||
await nocoCreateLookups(aTblSchema); |
||||
|
||||
// add roll-ups
|
||||
await nocoCreateRollups(aTblSchema); |
||||
|
||||
// configure primary values
|
||||
await nocoSetPrimary(aTblSchema); |
||||
|
||||
// await nc_DumpTableSchema();
|
||||
let ncTblList = await api.dbTable.list(global_ncCreatedProjectSchema.id); |
||||
for (let i = 0; i < ncTblList.list.length; i++) { |
||||
let ncTbl = await api.dbTable.read(ncTblList.list[i].id); |
||||
await nocoReadData(ncTbl, nocoBaseDataProcessing); |
||||
} |
||||
|
||||
// // Configure link @ Data row's
|
||||
for (let idx = 0; idx < global_ncLinkMappingTable.length; idx++) { |
||||
let x = global_ncLinkMappingTable[idx]; |
||||
let ncTbl = await nc_getTableSchema(aTbl_getTableName(x.aTbl.tblId).tn); |
||||
await nocoReadDataSelected(ncTbl, nocoLinkProcessing, x.aTbl.name); |
||||
} |
||||
} |
||||
|
||||
nc_migrateATbl().catch(e => { |
||||
console.log(e); |
||||
}); |
||||
@ -0,0 +1,63 @@
|
||||
{ |
||||
"compilerOptions": { |
||||
"skipLibCheck": false, |
||||
"composite": true, |
||||
"target": "es2017", |
||||
"outDir": "build", |
||||
"rootDir": "src", |
||||
"moduleResolution": "node", |
||||
"module": "commonjs", |
||||
"declaration": true, |
||||
"inlineSourceMap": true, |
||||
"esModuleInterop": true |
||||
/* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, |
||||
"allowJs": false, |
||||
// "strict": true /* Enable all strict type-checking options. */, |
||||
|
||||
/* Strict Type-Checking Options */ |
||||
// "noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */, |
||||
// "strictNullChecks": true /* Enable strict null checks. */, |
||||
// "strictFunctionTypes": true /* Enable strict checking of function types. */, |
||||
// "strictPropertyInitialization": true /* Enable strict checking of property initialization in classes. */, |
||||
// "noImplicitThis": true /* Raise error on 'this' expressions with an implied 'any' type. */, |
||||
// "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */, |
||||
"resolveJsonModule": true, |
||||
/* Additional Checks */ |
||||
// "noUnusedLocals": true |
||||
// /* Report errors on unused locals. */, |
||||
// "noUnusedParameters": true |
||||
// /* Report errors on unused parameters. */, |
||||
"noImplicitReturns": true |
||||
/* Report error when not all code paths in function return a value. */, |
||||
"noFallthroughCasesInSwitch": true |
||||
/* Report errors for fallthrough cases in switch statement. */, |
||||
/* Debugging Options */ |
||||
"traceResolution": false |
||||
/* Report module resolution log messages. */, |
||||
"listEmittedFiles": false |
||||
/* Print names of generated files part of the compilation. */, |
||||
"listFiles": false |
||||
/* Print names of files part of the compilation. */, |
||||
"pretty": true |
||||
/* Stylize errors and messages using color and context. */, |
||||
/* Experimental Options */ |
||||
// "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */, |
||||
// "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */, |
||||
|
||||
"lib": [ |
||||
"es2017", |
||||
"dom" |
||||
], |
||||
"types": [ |
||||
"node" |
||||
], |
||||
"typeRoots": [ |
||||
"../../../node_modules/@types" |
||||
] |
||||
}, |
||||
"include": [ |
||||
"src/**/*.ts", |
||||
"src/**/*.json" |
||||
], |
||||
"compileOnSave": false |
||||
} |
||||
Loading…
Reference in new issue