github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

fix: treat decimal as double in mysql 

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>
pull/2368/head
Raju Udava 2 years ago
parent
4b5a858586
commit
8f14a81125
1 changed files with 150 additions and 130 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 280
      packages/nocodb/src/lib/meta/api/sync/helpers/job.ts

280
packages/nocodb/src/lib/meta/api/sync/helpers/job.ts
Unescape View File

@ -28,7 +28,7 @@ export default async (
ncId: ncId, ncId: ncId,
ncParent: parent, ncParent: parent,
// name added to assist in quick debug // name added to assist in quick debug
ncName: ncName ncName: ncName,
}; };
}, },
@ -45,7 +45,7 @@ export default async (
// get nc-title from airtable ID // get nc-title from airtable ID
getNcNameFromAtId(aId) { getNcNameFromAtId(aId) {
return this.mapTbl[aId]?.ncName; return this.mapTbl[aId]?.ncName;
} },
}; };
function logBasic(log) { function logBasic(log) {
@ -99,20 +99,20 @@ export default async (
total: 0, total: 0,
grid: 0, grid: 0,
gallery: 0, gallery: 0,
form: 0 form: 0,
}, },
fetchAt: { fetchAt: {
count: 0, count: 0,
time: 0 time: 0,
}, },
migrationSkipLog: { migrationSkipLog: {
count: 0, count: 0,
log: [] log: [],
}, },
data: { data: {
records: 0, records: 0,
nestedLinks: 0 nestedLinks: 0,
} },
}; };
function updateMigrationSkipLog(tbl, col, type, reason?) { function updateMigrationSkipLog(tbl, col, type, reason?) {
@ -131,7 +131,7 @@ export default async (
if (!sDB.shareId) if (!sDB.shareId)
throw { throw {
message: message:
'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details' 'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
}; };
if (sDB.shareId.startsWith('exp')) { if (sDB.shareId.startsWith('exp')) {
@ -189,7 +189,7 @@ export default async (
lookup: UITypes.Lookup, lookup: UITypes.Lookup,
autoNumber: UITypes.AutoNumber, autoNumber: UITypes.AutoNumber,
barcode: UITypes.Barcode, barcode: UITypes.Barcode,
button: UITypes.Button button: UITypes.Button,
}; };
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
@ -211,8 +211,12 @@ export default async (
const col_alias = name.trim().replace(/\./g, '_'); const col_alias = name.trim().replace(/\./g, '_');
// check if already a column exists with same name? // check if already a column exists with same name?
const duplicateTitle = table.columns.find(x => x.title?.toLowerCase() === col_alias?.toLowerCase()); const duplicateTitle = table.columns.find(
const duplicateColumn = table.columns.find(x => x.column_name?.toLowerCase() === col_name?.toLowerCase()); (x) => x.title?.toLowerCase() === col_alias?.toLowerCase()
);
const duplicateColumn = table.columns.find(
(x) => x.column_name?.toLowerCase() === col_name?.toLowerCase()
);
if (duplicateTitle) { if (duplicateTitle) {
if (enableErrorLogs) console.log(`## Duplicate title ${col_alias}`); if (enableErrorLogs) console.log(`## Duplicate title ${col_alias}`);
} }
@ -220,7 +224,7 @@ export default async (
return { return {
// kludge: error observed in Nc with space around column-name // kludge: error observed in Nc with space around column-name
title: col_alias + (duplicateTitle ? '_2' : ''), title: col_alias + (duplicateTitle ? '_2' : ''),
column_name: col_name + (duplicateColumn ? '_2' : '') column_name: col_name + (duplicateColumn ? '_2' : ''),
}; };
} }
@ -228,15 +232,15 @@ export default async (
// //
// @ts-ignore // @ts-ignore
function aTbl_getTableName(tblId) { function aTbl_getTableName(tblId) {
const sheetObj = g_aTblSchema.find(tbl => tbl.id === tblId); const sheetObj = g_aTblSchema.find((tbl) => tbl.id === tblId);
return { return {
tn: sheetObj.name tn: sheetObj.name,
}; };
} }
const ncSchema = { const ncSchema = {
tables: [], tables: [],
tablesById: {} tablesById: {},
}; };
// aTbl: retrieve column name from column ID // aTbl: retrieve column name from column ID
@ -244,11 +248,11 @@ export default async (
function aTbl_getColumnName(colId): any { function aTbl_getColumnName(colId): any {
for (let i = 0; i < g_aTblSchema.length; i++) { for (let i = 0; i < g_aTblSchema.length; i++) {
const sheetObj = g_aTblSchema[i]; const sheetObj = g_aTblSchema[i];
const column = sheetObj.columns.find(col => col.id === colId); const column = sheetObj.columns.find((col) => col.id === colId);
if (column !== undefined) if (column !== undefined)
return { return {
tn: sheetObj.name, tn: sheetObj.name,
cn: column.name cn: column.name,
}; };
} }
} }
@ -283,7 +287,7 @@ export default async (
// not migrated column, skip // not migrated column, skip
if (ncColId === undefined || ncTblId === undefined) return 0; if (ncColId === undefined || ncTblId === undefined) return 0;
return ncSchema.tablesById[ncTblId].columns.find(x => x.id === ncColId); return ncSchema.tablesById[ncTblId].columns.find((x) => x.id === ncColId);
} }
// retrieve nc table schema using table name // retrieve nc table schema using table name
@ -295,12 +299,12 @@ export default async (
// let ncTbl = await api.dbTable.read(ncTblId); // let ncTbl = await api.dbTable.read(ncTblId);
// return ncTbl; // return ncTbl;
return ncSchema.tables.find(x => x.title === tableName); return ncSchema.tables.find((x) => x.title === tableName);
} }
// delete project if already exists // delete project if already exists
async function init({ async function init({
projectName projectName,
}: { }: {
projectName?: string; projectName?: string;
projectId?: string; projectId?: string;
@ -308,7 +312,7 @@ export default async (
// delete 'sample' project if already exists // delete 'sample' project if already exists
const x = await api.project.list(); const x = await api.project.list();
const sampleProj = x.list.find(a => a.title === projectName); const sampleProj = x.list.find((a) => a.title === projectName);
if (sampleProj) { if (sampleProj) {
await api.project.delete(sampleProj.id); await api.project.delete(sampleProj.id);
} }
@ -387,7 +391,7 @@ export default async (
// const csvOpt = "'" + opt.join("','") + "'"; // const csvOpt = "'" + opt.join("','") + "'";
const optSansDuplicate = [...new Set(opt)]; const optSansDuplicate = [...new Set(opt)];
const csvOpt = optSansDuplicate const csvOpt = optSansDuplicate
.map(v => `'${v.replace(/'/g, "\\'").replace(/,/g, '.')}'`) .map((v) => `'${v.replace(/'/g, "\\'").replace(/,/g, '.')}'`)
.join(','); .join(',');
return { type: 'select', data: csvOpt }; return { type: 'select', data: csvOpt };
} }
@ -434,15 +438,15 @@ export default async (
column_name: ncSysFields.id, column_name: ncSysFields.id,
uidt: UITypes.ID, uidt: UITypes.ID,
meta: { meta: {
ag: 'nc' ag: 'nc',
} },
}, },
{ {
title: ncSysFields.hash, title: ncSysFields.hash,
column_name: ncSysFields.hash, column_name: ncSysFields.hash,
uidt: UITypes.SingleLineText, uidt: UITypes.SingleLineText,
system: true system: true,
} },
]; ];
for (let j = 0; j < tblSchema[i].columns.length; j++) { for (let j = 0; j < tblSchema[i].columns.length; j++) {
@ -459,7 +463,7 @@ export default async (
// Enable to use aTbl identifiers as is: id: col.id, // Enable to use aTbl identifiers as is: id: col.id,
title: ncName.title, title: ncName.title,
column_name: uniqueColNameGen(ncName.column_name), column_name: uniqueColNameGen(ncName.column_name),
uidt: getNocoType(col) uidt: getNocoType(col),
}; };
// not supported datatype: pure formula field // not supported datatype: pure formula field
@ -484,6 +488,11 @@ export default async (
// change from default 'tinytext' as airtable allows more than 255 characters // change from default 'tinytext' as airtable allows more than 255 characters
// for single line text column type // for single line text column type
if (col.type === 'text') ncCol.dt = 'text'; if (col.type === 'text') ncCol.dt = 'text';
if (ncCol.uidt === UITypes.Decimal) {
ncCol.dt = 'double';
ncCol.dtxp = 22;
ncCol.dtxs = '2';
}
// additional column parameters when applicable // additional column parameters when applicable
const colOptions = getNocoTypeOptions(col); const colOptions = getNocoTypeOptions(col);
@ -529,7 +538,8 @@ export default async (
await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title); await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title);
for (let colIdx = 0; colIdx < table.columns.length; colIdx++) { for (let colIdx = 0; colIdx < table.columns.length; colIdx++) {
const aId = aTblSchema[idx].columns.find( const aId = aTblSchema[idx].columns.find(
x => x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title (x) =>
x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title
)?.id; )?.id;
if (aId) if (aId)
await sMap.addToMappingTbl( await sMap.addToMappingTbl(
@ -546,11 +556,11 @@ export default async (
const view = await api.dbView.list(table.id); const view = await api.dbView.list(table.id);
recordPerfStats(_perfStart, 'dbView.list'); recordPerfStats(_perfStart, 'dbView.list');
const aTbl_grid = aTblSchema[idx].views.find(x => x.type === 'grid'); const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid');
logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`);
_perfStart = recordPerfStart(); _perfStart = recordPerfStart();
await api.dbView.update(view.list[0].id, { await api.dbView.update(view.list[0].id, {
title: aTbl_grid.name title: aTbl_grid.name,
}); });
recordPerfStats(_perfStart, 'dbView.update'); recordPerfStats(_perfStart, 'dbView.update');
@ -573,7 +583,7 @@ export default async (
// Link to another RECORD // Link to another RECORD
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const aTblLinkColumns = aTblSchema[idx].columns.filter( const aTblLinkColumns = aTblSchema[idx].columns.filter(
x => x.type === 'foreignKey' (x) => x.type === 'foreignKey'
); );
// Link columns exist // Link columns exist
@ -640,7 +650,7 @@ export default async (
column_name: ncName.column_name, column_name: ncName.column_name,
parentId: srcTableId, parentId: srcTableId,
childId: childTableId, childId: childTableId,
type: 'mm' type: 'mm',
// aTblLinkColumns[i].typeOptions.relationship === 'many' // aTblLinkColumns[i].typeOptions.relationship === 'many'
// ? 'mm' // ? 'mm'
// : 'hm' // : 'hm'
@ -649,7 +659,9 @@ export default async (
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === ncName.title)?.id; const ncId = ncTbl.columns.find(
(x) => x.title === ncName.title
)?.id;
await sMap.addToMappingTbl( await sMap.addToMappingTbl(
aTblLinkColumns[i].id, aTblLinkColumns[i].id,
ncId, ncId,
@ -664,12 +676,12 @@ export default async (
title: ncName.title, title: ncName.title,
parentId: srcTableId, parentId: srcTableId,
childId: childTableId, childId: childTableId,
type: 'mm' type: 'mm',
}, },
aTbl: { aTbl: {
tblId: aTblSchema[idx].id, tblId: aTblSchema[idx].id,
...aTblLinkColumns[i] ...aTblLinkColumns[i],
} },
}; };
ncLinkMappingTable.push(link); ncLinkMappingTable.push(link);
@ -682,7 +694,7 @@ export default async (
// 3. using foreign parent & child column ID, find associated mapping in child table // 3. using foreign parent & child column ID, find associated mapping in child table
// 4. update column name // 4. update column name
const x = ncLinkMappingTable.findIndex( const x = ncLinkMappingTable.findIndex(
x => (x) =>
x.aTbl.tblId === x.aTbl.tblId ===
aTblLinkColumns[i].typeOptions.foreignTableId && aTblLinkColumns[i].typeOptions.foreignTableId &&
x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId
@ -705,7 +717,7 @@ export default async (
// let parentTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.parentId] // let parentTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.parentId]
let parentLinkColumn = parentTblSchema.columns.find( let parentLinkColumn = parentTblSchema.columns.find(
col => col.title === ncLinkMappingTable[x].nc.title (col) => col.title === ncLinkMappingTable[x].nc.title
); );
if (parentLinkColumn === undefined) { if (parentLinkColumn === undefined) {
@ -721,7 +733,7 @@ export default async (
// hack // fix me // hack // fix me
if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') { if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') {
parentLinkColumn = parentTblSchema.columns.find( parentLinkColumn = parentTblSchema.columns.find(
col => col.title === ncLinkMappingTable[x].nc.title + '_2' (col) => col.title === ncLinkMappingTable[x].nc.title + '_2'
); );
} }
@ -732,7 +744,7 @@ export default async (
// mapping between child & parent column id is direct // mapping between child & parent column id is direct
// //
childLinkColumn = childTblSchema.columns.find( childLinkColumn = childTblSchema.columns.find(
col => (col) =>
col.uidt === UITypes.LinkToAnotherRecord && col.uidt === UITypes.LinkToAnotherRecord &&
col.colOptions.fk_child_column_id === col.colOptions.fk_child_column_id ===
parentLinkColumn.colOptions.fk_child_column_id && parentLinkColumn.colOptions.fk_child_column_id &&
@ -744,7 +756,7 @@ export default async (
// mapping between child & parent column id is inverted // mapping between child & parent column id is inverted
// //
childLinkColumn = childTblSchema.columns.find( childLinkColumn = childTblSchema.columns.find(
col => (col) =>
col.uidt === UITypes.LinkToAnotherRecord && col.uidt === UITypes.LinkToAnotherRecord &&
col.colOptions.fk_child_column_id === col.colOptions.fk_child_column_id ===
parentLinkColumn.colOptions.fk_parent_column_id && parentLinkColumn.colOptions.fk_parent_column_id &&
@ -757,7 +769,7 @@ export default async (
// check if already a column exists with this name? // check if already a column exists with this name?
const duplicate = childTblSchema.columns.find( const duplicate = childTblSchema.columns.find(
x => x.title === aTblLinkColumns[i].name (x) => x.title === aTblLinkColumns[i].name
); );
const suffix = duplicate ? '_2' : ''; const suffix = duplicate ? '_2' : '';
if (duplicate) if (duplicate)
@ -781,7 +793,7 @@ export default async (
{ {
...childLinkColumn, ...childLinkColumn,
title: ncName.title, title: ncName.title,
column_name: ncName.column_name column_name: ncName.column_name,
} }
); );
recordPerfStats(_perfStart, 'dbTableColumn.update'); recordPerfStats(_perfStart, 'dbTableColumn.update');
@ -789,7 +801,7 @@ export default async (
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find( const ncId = ncTbl.columns.find(
x => x.title === aTblLinkColumns[i].name + suffix (x) => x.title === aTblLinkColumns[i].name + suffix
)?.id; )?.id;
await sMap.addToMappingTbl( await sMap.addToMappingTbl(
aTblLinkColumns[i].id, aTblLinkColumns[i].id,
@ -809,7 +821,7 @@ export default async (
// LookUps // LookUps
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const aTblColumns = aTblSchema[idx].columns.filter( const aTblColumns = aTblSchema[idx].columns.filter(
x => x.type === 'lookup' (x) => x.type === 'lookup'
); );
// parent table ID // parent table ID
@ -821,8 +833,9 @@ export default async (
// Lookup // Lookup
for (let i = 0; i < aTblColumns.length; i++) { for (let i = 0; i < aTblColumns.length; i++) {
logDetailed( logDetailed(
`[${idx + 1}/${aTblSchema.length}] Configuring Lookup :: [${i + `[${idx + 1}/${aTblSchema.length}] Configuring Lookup :: [${
1}/${aTblColumns.length}] ${aTblSchema[idx].name}` i + 1
}/${aTblColumns.length}] ${aTblSchema[idx].name}`
); );
// something is not right, skip // something is not right, skip
@ -869,14 +882,15 @@ export default async (
title: ncName.title, title: ncName.title,
column_name: ncName.column_name, column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId, fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId fk_lookup_column_id: ncLookupColumnId,
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create'); recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name) const ncId = ncTbl.columns.find(
?.id; (x) => x.title === aTblColumns[i].name
)?.id;
await sMap.addToMappingTbl( await sMap.addToMappingTbl(
aTblColumns[i].id, aTblColumns[i].id,
ncId, ncId,
@ -945,14 +959,14 @@ export default async (
title: ncName.title, title: ncName.title,
column_name: ncName.column_name, column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId, fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId fk_lookup_column_id: ncLookupColumnId,
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create'); recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find( const ncId = ncTbl.columns.find(
x => x.title === nestedLookupTbl[0].name (x) => x.title === nestedLookupTbl[0].name
)?.id; )?.id;
await sMap.addToMappingTbl( await sMap.addToMappingTbl(
nestedLookupTbl[0].id, nestedLookupTbl[0].id,
@ -984,7 +998,7 @@ export default async (
MIN: 'min', MIN: 'min',
OR: '', OR: '',
SUM: 'sum', SUM: 'sum',
XOR: '' XOR: '',
}; };
return aTbl_ncRollUp[fn]; return aTbl_ncRollUp[fn];
} }
@ -994,7 +1008,7 @@ export default async (
// Rollup // Rollup
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const aTblColumns = aTblSchema[idx].columns.filter( const aTblColumns = aTblSchema[idx].columns.filter(
x => x.type === 'rollup' (x) => x.type === 'rollup'
); );
// parent table ID // parent table ID
@ -1006,8 +1020,9 @@ export default async (
// rollup exist // rollup exist
for (let i = 0; i < aTblColumns.length; i++) { for (let i = 0; i < aTblColumns.length; i++) {
logDetailed( logDetailed(
`[${idx + 1}/${aTblSchema.length}] Configuring Rollup :: [${i + `[${idx + 1}/${aTblSchema.length}] Configuring Rollup :: [${
1}/${aTblColumns.length}] ${aTblSchema[idx].name}` i + 1
}/${aTblColumns.length}] ${aTblSchema[idx].name}`
); );
// fetch associated rollup function // fetch associated rollup function
@ -1088,14 +1103,15 @@ export default async (
column_name: ncName.column_name, column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId, fk_relation_column_id: ncRelationColumnId,
fk_rollup_column_id: ncRollupColumnId, fk_rollup_column_id: ncRollupColumnId,
rollup_function: ncRollupFn rollup_function: ncRollupFn,
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create'); recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name) const ncId = ncTbl.columns.find(
?.id; (x) => x.title === aTblColumns[i].name
)?.id;
await sMap.addToMappingTbl( await sMap.addToMappingTbl(
aTblColumns[i].id, aTblColumns[i].id,
ncId, ncId,
@ -1144,14 +1160,15 @@ export default async (
title: ncName.title, title: ncName.title,
column_name: ncName.column_name, column_name: ncName.column_name,
fk_relation_column_id: ncRelationColumnId, fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId fk_lookup_column_id: ncLookupColumnId,
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create'); recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === nestedLookupTbl[0].name) const ncId = ncTbl.columns.find(
?.id; (x) => x.title === nestedLookupTbl[0].name
)?.id;
await sMap.addToMappingTbl( await sMap.addToMappingTbl(
nestedLookupTbl[0].id, nestedLookupTbl[0].id,
ncId, ncId,
@ -1206,7 +1223,7 @@ export default async (
recordPerfStats(_perfStart, 'dbView.gridColumnsList'); recordPerfStats(_perfStart, 'dbView.gridColumnsList');
} }
return viewDetails.find(x => x.fk_column_id === ncColumnId)?.id; return viewDetails.find((x) => x.fk_column_id === ncColumnId)?.id;
} }
////////// Data processing ////////// Data processing
@ -1218,7 +1235,7 @@ export default async (
// kludge - // kludge -
// trim spaces on either side of column name // trim spaces on either side of column name
// leads to error in NocoDB // leads to error in NocoDB
Object.keys(rec).forEach(key => { Object.keys(rec).forEach((key) => {
const replacedKey = key.trim().replace(/\./g, '_'); const replacedKey = key.trim().replace(/\./g, '_');
if (key !== replacedKey) { if (key !== replacedKey) {
rec[replacedKey] = rec[key]; rec[replacedKey] = rec[key];
@ -1229,7 +1246,7 @@ export default async (
// post-processing on the record // post-processing on the record
for (const [key, value] of Object.entries(rec as { [key: string]: any })) { for (const [key, value] of Object.entries(rec as { [key: string]: any })) {
// retrieve datatype // retrieve datatype
const dt = table.columns.find(x => x.title === key)?.uidt; const dt = table.columns.find((x) => x.title === key)?.uidt;
switch (dt) { switch (dt) {
// https://www.npmjs.com/package/validator // https://www.npmjs.com/package/validator
@ -1246,7 +1263,7 @@ export default async (
if (ncLinkDataStore[table.title][record.id] === undefined) if (ncLinkDataStore[table.title][record.id] === undefined)
ncLinkDataStore[table.title][record.id] = { ncLinkDataStore[table.title][record.id] = {
id: record.id, id: record.id,
fields: {} fields: {},
}; };
ncLinkDataStore[table.title][record.id]['fields'][key] = value; ncLinkDataStore[table.title][record.id]['fields'][key] = value;
} }
@ -1281,9 +1298,7 @@ export default async (
case UITypes.DateTime: case UITypes.DateTime:
case UITypes.CreateTime: case UITypes.CreateTime:
case UITypes.LastModifiedTime: case UITypes.LastModifiedTime:
rec[key] = dayjs(value) rec[key] = dayjs(value).utc().format('YYYY-MM-DD HH:mm');
.utc()
.format('YYYY-MM-DD HH:mm');
break; break;
case UITypes.Date: case UITypes.Date:
@ -1292,9 +1307,7 @@ export default async (
rec[key] = null; rec[key] = null;
logBasic(`:: Invalid date ${value}`); logBasic(`:: Invalid date ${value}`);
} else { } else {
rec[key] = dayjs(value) rec[key] = dayjs(value).utc().format('YYYY-MM-DD');
.utc()
.format('YYYY-MM-DD');
} }
break; break;
@ -1303,7 +1316,7 @@ export default async (
break; break;
case UITypes.MultiSelect: case UITypes.MultiSelect:
rec[key] = value.map(v => `${v.replace(/,/g, '.')}`).join(','); rec[key] = value.map((v) => `${v.replace(/,/g, '.')}`).join(',');
break; break;
case UITypes.Attachment: case UITypes.Attachment:
@ -1314,18 +1327,18 @@ export default async (
try { try {
logBasic( logBasic(
` :: Retrieving attachment :: ${value ` :: Retrieving attachment :: ${value
?.map(a => a.filename?.split('?')?.[0]) ?.map((a) => a.filename?.split('?')?.[0])
.join(', ')}` .join(', ')}`
); );
tempArr = await api.storage.uploadByUrl( tempArr = await api.storage.uploadByUrl(
{ {
path: `noco/${sDB.projectName}/${table.title}/${key}` path: `noco/${sDB.projectName}/${table.title}/${key}`,
}, },
value?.map(attachment => ({ value?.map((attachment) => ({
fileName: attachment.filename?.split('?')?.[0], fileName: attachment.filename?.split('?')?.[0],
url: attachment.url, url: attachment.url,
size: attachment.size, size: attachment.size,
mimetype: attachment.type mimetype: attachment.type,
})) }))
); );
} catch (e) { } catch (e) {
@ -1355,7 +1368,7 @@ export default async (
.select({ .select({
pageSize: 100, pageSize: 100,
// maxRecords: 100, // maxRecords: 100,
fields: fields fields: fields,
}) })
.eachPage( .eachPage(
async function page(records, fetchNextPage) { async function page(records, fetchNextPage) {
@ -1364,11 +1377,12 @@ export default async (
// This function (`page`) will get called for each page of records. // This function (`page`) will get called for each page of records.
// records.forEach(record => callback(table, record)); // records.forEach(record => callback(table, record));
logBasic( logBasic(
`:: ${table.title} / ${fields} : ${recordCnt + `:: ${table.title} / ${fields} : ${
1} ~ ${(recordCnt += 100)}` recordCnt + 1
} ~ ${(recordCnt += 100)}`
); );
await Promise.all( await Promise.all(
records.map(r => callback(projName, table, r, fields)) records.map((r) => callback(projName, table, r, fields))
); );
// To fetch the next page of records, call `fetchNextPage`. // To fetch the next page of records, call `fetchNextPage`.
@ -1391,7 +1405,7 @@ export default async (
function nc_isLinkExists(airtableFieldId) { function nc_isLinkExists(airtableFieldId) {
return !!ncLinkMappingTable.find( return !!ncLinkMappingTable.find(
x => x.aTbl.typeOptions.symmetricColumnId === airtableFieldId (x) => x.aTbl.typeOptions.symmetricColumnId === airtableFieldId
); );
} }
@ -1400,7 +1414,7 @@ export default async (
logDetailed(`Create Project: ${projName}`); logDetailed(`Create Project: ${projName}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
ncCreatedProjectSchema = await api.project.create({ ncCreatedProjectSchema = await api.project.create({
title: projName title: projName,
}); });
recordPerfStats(_perfStart, 'project.create'); recordPerfStats(_perfStart, 'project.create');
} }
@ -1418,7 +1432,7 @@ export default async (
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id; const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
const galleryViews = aTblSchema[idx].views.filter( const galleryViews = aTblSchema[idx].views.filter(
x => x.type === 'gallery' (x) => x.type === 'gallery'
); );
const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form;
@ -1430,7 +1444,7 @@ export default async (
// create view // create view
await getViewData(galleryViews[i].id); await getViewData(galleryViews[i].id);
const viewName = aTblSchema[idx].views.find( const viewName = aTblSchema[idx].views.find(
x => x.id === galleryViews[i].id (x) => x.id === galleryViews[i].id
)?.name; )?.name;
logBasic( logBasic(
@ -1456,7 +1470,7 @@ export default async (
if (!sDB.options.syncViews) return; if (!sDB.options.syncViews) return;
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
const formViews = aTblSchema[idx].views.filter(x => x.type === 'form'); const formViews = aTblSchema[idx].views.filter((x) => x.type === 'form');
const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form;
rtc.view.form += formViews.length; rtc.view.form += formViews.length;
@ -1466,7 +1480,7 @@ export default async (
// create view // create view
const vData = await getViewData(formViews[i].id); const vData = await getViewData(formViews[i].id);
const viewName = aTblSchema[idx].views.find( const viewName = aTblSchema[idx].views.find(
x => x.id === formViews[i].id (x) => x.id === formViews[i].id
)?.name; )?.name;
logBasic( logBasic(
@ -1497,7 +1511,7 @@ export default async (
subheading: desc, subheading: desc,
success_msg: msg, success_msg: msg,
submit_another_form: refreshMode.includes('REFRESH_BUTTON'), submit_another_form: refreshMode.includes('REFRESH_BUTTON'),
show_blank_form: refreshMode.includes('AUTO_REFRESH') show_blank_form: refreshMode.includes('AUTO_REFRESH'),
}; };
logDetailed(`NC API dbView.formCreate :: ${viewName}`); logDetailed(`NC API dbView.formCreate :: ${viewName}`);
@ -1528,7 +1542,7 @@ export default async (
async function nocoConfigureGridView(sDB, aTblSchema) { async function nocoConfigureGridView(sDB, aTblSchema) {
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
const gridViews = aTblSchema[idx].views.filter(x => x.type === 'grid'); const gridViews = aTblSchema[idx].views.filter((x) => x.type === 'grid');
let viewCnt = idx; let viewCnt = idx;
if (syncDB.options.syncViews) if (syncDB.options.syncViews)
@ -1542,13 +1556,13 @@ export default async (
// retrieve view name & associated NC-ID // retrieve view name & associated NC-ID
const viewName = aTblSchema[idx].views.find( const viewName = aTblSchema[idx].views.find(
x => x.id === gridViews[i].id (x) => x.id === gridViews[i].id
)?.name; )?.name;
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const viewList: any = await api.dbView.list(tblId); const viewList: any = await api.dbView.list(tblId);
recordPerfStats(_perfStart, 'dbView.list'); recordPerfStats(_perfStart, 'dbView.list');
let ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id; let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id;
logBasic( logBasic(
`:: [${viewCnt + i + 1}/${rtc.view.total}] Grid : ${ `:: [${viewCnt + i + 1}/${rtc.view.total}] Grid : ${
@ -1561,7 +1575,7 @@ export default async (
logDetailed(`NC API dbView.gridCreate :: ${viewName}`); logDetailed(`NC API dbView.gridCreate :: ${viewName}`);
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
const viewCreated = await api.dbView.gridCreate(tblId, { const viewCreated = await api.dbView.gridCreate(tblId, {
title: viewName title: viewName,
}); });
recordPerfStats(_perfStart, 'dbView.gridCreate'); recordPerfStats(_perfStart, 'dbView.gridCreate');
@ -1592,7 +1606,7 @@ export default async (
logDetailed(` Configure filter set`); logDetailed(` Configure filter set`);
// skip filters if nested // skip filters if nested
if (!vData.filters.filterSet.find(x => x?.type === 'nested')) { if (!vData.filters.filterSet.find((x) => x?.type === 'nested')) {
await nc_configureFilters(ncViewId, vData.filters); await nc_configureFilters(ncViewId, vData.filters);
} }
} }
@ -1614,7 +1628,7 @@ export default async (
edit: 'editor', edit: 'editor',
comment: 'commenter', comment: 'commenter',
read: 'viewer', read: 'viewer',
none: 'viewer' none: 'viewer',
}; };
const userList = aTblSchema.appBlanket.userInfoById; const userList = aTblSchema.appBlanket.userInfoById;
const totalUsers = Object.keys(userList).length; const totalUsers = Object.keys(userList).length;
@ -1629,11 +1643,16 @@ export default async (
); );
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
insertJobs.push( insertJobs.push(
api.auth.projectUserAdd(ncCreatedProjectSchema.id, { api.auth
.projectUserAdd(ncCreatedProjectSchema.id, {
email: value.email, email: value.email,
roles: userRoles[value.permissionLevel] roles: userRoles[value.permissionLevel],
}) })
.catch(e => e.response?.data?.msg ? logBasic(`NOTICE: ${e.response.data.msg}`) : console.log(e)) .catch((e) =>
e.response?.data?.msg
? logBasic(`NOTICE: ${e.response.data.msg}`)
: console.log(e)
)
); );
recordPerfStats(_perfStart, 'auth.projectUserAdd'); recordPerfStats(_perfStart, 'auth.projectUserAdd');
} }
@ -1644,7 +1663,7 @@ export default async (
const tblId = tblSchema.id; const tblId = tblSchema.id;
// replace entry from array if already exists // replace entry from array if already exists
const idx = ncSchema.tables.findIndex(x => x.id === tblId); const idx = ncSchema.tables.findIndex((x) => x.id === tblId);
if (idx !== -1) ncSchema.tables.splice(idx, 1); if (idx !== -1) ncSchema.tables.splice(idx, 1);
ncSchema.tables.push(tblSchema); ncSchema.tables.push(tblSchema);
@ -1673,27 +1692,27 @@ export default async (
columns: 0, columns: 0,
links: 0, links: 0,
lookup: 0, lookup: 0,
rollup: 0 rollup: 0,
}, },
nc: { nc: {
columns: 0, columns: 0,
links: 0, links: 0,
lookup: 0, lookup: 0,
rollup: 0, rollup: 0,
invalidColumn: 0 invalidColumn: 0,
} },
}; };
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
migrationStatsObj.table_name = aTblSchema[idx].name; migrationStatsObj.table_name = aTblSchema[idx].name;
const aTblLinkColumns = aTblSchema[idx].columns.filter( const aTblLinkColumns = aTblSchema[idx].columns.filter(
x => x.type === 'foreignKey' (x) => x.type === 'foreignKey'
); );
const aTblLookup = aTblSchema[idx].columns.filter( const aTblLookup = aTblSchema[idx].columns.filter(
x => x.type === 'lookup' (x) => x.type === 'lookup'
); );
const aTblRollup = aTblSchema[idx].columns.filter( const aTblRollup = aTblSchema[idx].columns.filter(
x => x.type === 'rollup' (x) => x.type === 'rollup'
); );
let invalidColumnId = 0; let invalidColumnId = 0;
@ -1719,10 +1738,10 @@ export default async (
const ncTbl = await nc_getTableSchema(aTblSchema[idx].name); const ncTbl = await nc_getTableSchema(aTblSchema[idx].name);
const linkColumn = ncTbl.columns.filter( const linkColumn = ncTbl.columns.filter(
x => x.uidt === UITypes.LinkToAnotherRecord (x) => x.uidt === UITypes.LinkToAnotherRecord
); );
const lookup = ncTbl.columns.filter(x => x.uidt === UITypes.Lookup); const lookup = ncTbl.columns.filter((x) => x.uidt === UITypes.Lookup);
const rollup = ncTbl.columns.filter(x => x.uidt === UITypes.Rollup); const rollup = ncTbl.columns.filter((x) => x.uidt === UITypes.Rollup);
// all links hardwired as m2m. m2m generates additional tables per link // all links hardwired as m2m. m2m generates additional tables per link
// hence link/2 // hence link/2
@ -1778,7 +1797,7 @@ export default async (
} }
jsonfile.writeFileSync('stats.csv', perflog, { spaces: 2 }); jsonfile.writeFileSync('stats.csv', perflog, { spaces: 2 });
jsonfile.writeFileSync('skip.txt', rtc.migrationSkipLog.log, { jsonfile.writeFileSync('skip.txt', rtc.migrationSkipLog.log, {
spaces: 2 spaces: 2,
}); });
} }
@ -1798,16 +1817,16 @@ export default async (
total: rtc.view.total, total: rtc.view.total,
grid: rtc.view.grid, grid: rtc.view.grid,
gallery: rtc.view.gallery, gallery: rtc.view.gallery,
form: rtc.view.form form: rtc.view.form,
}, },
axios: { axios: {
count: rtc.fetchAt.count, count: rtc.fetchAt.count,
time: rtc.fetchAt.time time: rtc.fetchAt.time,
}, },
totalRecords: rtc.data.records, totalRecords: rtc.data.records,
nestedLinks: rtc.data.nestedLinks nestedLinks: rtc.data.nestedLinks,
} },
} },
}); });
} }
@ -1826,7 +1845,7 @@ export default async (
contains: 'like', contains: 'like',
doesNotContain: 'nlike', doesNotContain: 'nlike',
isAnyOf: 'eq', isAnyOf: 'eq',
isNoneOf: 'neq' isNoneOf: 'neq',
}; };
async function nc_configureFilters(viewId, f) { async function nc_configureFilters(viewId, f) {
@ -1873,7 +1892,7 @@ export default async (
fk_column_id: columnId, fk_column_id: columnId,
logical_op: f.conjunction, logical_op: f.conjunction,
comparison_op: filterMap[filter.operator], comparison_op: filterMap[filter.operator],
value: sMap.getNcNameFromAtId(filter.value[i]) value: sMap.getNcNameFromAtId(filter.value[i]),
}; };
ncFilters.push(fx); ncFilters.push(fx);
} }
@ -1884,7 +1903,7 @@ export default async (
fk_column_id: columnId, fk_column_id: columnId,
logical_op: f.conjunction, logical_op: f.conjunction,
comparison_op: filterMap[filter.operator], comparison_op: filterMap[filter.operator],
value: sMap.getNcNameFromAtId(filter.value) value: sMap.getNcNameFromAtId(filter.value),
}; };
ncFilters.push(fx); ncFilters.push(fx);
} }
@ -1896,7 +1915,7 @@ export default async (
fk_column_id: columnId, fk_column_id: columnId,
logical_op: f.conjunction, logical_op: f.conjunction,
comparison_op: filterMap[filter.operator], comparison_op: filterMap[filter.operator],
value: filter.value value: filter.value,
}; };
ncFilters.push(fx); ncFilters.push(fx);
} }
@ -1905,7 +1924,7 @@ export default async (
for (let i = 0; i < ncFilters.length; i++) { for (let i = 0; i < ncFilters.length; i++) {
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
await api.dbTableFilter.create(viewId, { await api.dbTableFilter.create(viewId, {
...ncFilters[i] ...ncFilters[i],
}); });
recordPerfStats(_perfStart, 'dbTableFilter.create'); recordPerfStats(_perfStart, 'dbTableFilter.create');
@ -1922,7 +1941,7 @@ export default async (
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
await api.dbTableSort.create(viewId, { await api.dbTableSort.create(viewId, {
fk_column_id: columnId, fk_column_id: columnId,
direction: s.sortSet[i].ascending ? 'asc' : 'dsc' direction: s.sortSet[i].ascending ? 'asc' : 'dsc',
}); });
recordPerfStats(_perfStart, 'dbTableSort.create'); recordPerfStats(_perfStart, 'dbTableSort.create');
} }
@ -1939,7 +1958,7 @@ export default async (
// retrieve table schema // retrieve table schema
const ncTbl = await nc_getTableSchema(tblName); const ncTbl = await nc_getTableSchema(tblName);
// retrieve view ID // retrieve view ID
const viewId = ncTbl.views.find(x => x.title === viewName).id; const viewId = ncTbl.views.find((x) => x.title === viewName).id;
let viewDetails; let viewDetails;
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
@ -1956,10 +1975,11 @@ export default async (
// nc-specific columns; default hide. // nc-specific columns; default hide.
for (let j = 0; j < hiddenColumns.length; j++) { for (let j = 0; j < hiddenColumns.length; j++) {
const ncColumnId = ncTbl.columns.find(x => x.title === hiddenColumns[j]) const ncColumnId = ncTbl.columns.find(
.id; (x) => x.title === hiddenColumns[j]
).id;
const ncViewColumnId = viewDetails.find( const ncViewColumnId = viewDetails.find(
x => x.fk_column_id === ncColumnId (x) => x.fk_column_id === ncColumnId
)?.id; )?.id;
// const ncViewColumnId = await nc_getViewColumnId( // const ncViewColumnId = await nc_getViewColumnId(
// viewId, // viewId,
@ -1972,7 +1992,7 @@ export default async (
const _perfStart = recordPerfStart(); const _perfStart = recordPerfStart();
await api.dbViewColumn.update(viewId, ncViewColumnId, { await api.dbViewColumn.update(viewId, ncViewColumnId, {
show: false, show: false,
order: j + 1 + c.length order: j + 1 + c.length,
}); });
recordPerfStats(_perfStart, 'dbViewColumn.update'); recordPerfStats(_perfStart, 'dbViewColumn.update');
} }
@ -2014,8 +2034,8 @@ export default async (
api = new Api({ api = new Api({
baseURL: syncDB.baseURL, baseURL: syncDB.baseURL,
headers: { headers: {
'xc-auth': syncDB.authToken 'xc-auth': syncDB.authToken,
} },
}); });
logDetailed('Project initialization started'); logDetailed('Project initialization started');
@ -2109,7 +2129,7 @@ export default async (
recordPerfStats(_perfStart, 'dbTable.read'); recordPerfStats(_perfStart, 'dbTable.read');
// not a migrated table, skip // not a migrated table, skip
if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) if (undefined === aTblSchema.find((x) => x.name === ncTbl.title))
continue; continue;
recordCnt = 0; recordCnt = 0;
@ -2123,7 +2143,7 @@ export default async (
logBasic, logBasic,
nocoBaseDataProcessing_v2, nocoBaseDataProcessing_v2,
sDB: syncDB, sDB: syncDB,
logDetailed logDetailed,
}); });
rtc.data.records += recordsMap[ncTbl.id].length; rtc.data.records += recordsMap[ncTbl.id].length;
@ -2143,7 +2163,7 @@ export default async (
insertedAssocRef, insertedAssocRef,
logDetailed, logDetailed,
records: recordsMap[ncTbl.id], records: recordsMap[ncTbl.id],
atNcAliasRef atNcAliasRef,
}); });
} }
@ -2213,7 +2233,7 @@ export default async (
if (e.response?.data?.msg) { if (e.response?.data?.msg) {
Tele.event({ Tele.event({
event: 'a:airtable-import:error', event: 'a:airtable-import:error',
data: { error: e.response.data.msg } data: { error: e.response.data.msg },
}); });
throw new Error(e.response.data.msg); throw new Error(e.response.data.msg);
} }

Write Preview
Loading…
Cancel
Save