{{ msg }}
@@ -424,7 +387,7 @@ onBeforeUnmount(() => {
v-if="
!progress ||
!progress.length ||
- (progress[progress.length - 1].status !== 'COMPLETED' && progress[progress.length - 1].status !== 'FAILED')
+ (progress[progress.length - 1].status !== 'completed' && progress[progress.length - 1].status !== 'failed')
"
class="flex items-center"
>
diff --git a/packages/nc-gui/pages/index/index/index.vue b/packages/nc-gui/pages/index/index/index.vue
index 18b4732536..c00946c751 100644
--- a/packages/nc-gui/pages/index/index/index.vue
+++ b/packages/nc-gui/pages/index/index/index.vue
@@ -95,7 +95,7 @@ const duplicateProject = (project: ProjectType) => {
await loadProjects()
- $jobs.subscribe(jobData.name, jobData.id, async (data: { status: string }) => {
+ $jobs.subscribe({ name: jobData.name, id: jobData.id }, null, async (data: { status: string }) => {
if (data.status === 'completed') {
await loadProjects()
} else if (data.status === 'failed') {
diff --git a/packages/nc-gui/plugins/jobs.ts b/packages/nc-gui/plugins/jobs.ts
index b672da60d1..ceede03259 100644
--- a/packages/nc-gui/plugins/jobs.ts
+++ b/packages/nc-gui/plugins/jobs.ts
@@ -6,6 +6,7 @@ export default defineNuxtPlugin(async (nuxtApp) => {
const { appInfo } = $(useGlobal())
let socket: Socket | null = null
+ let messageIndex = 0
const init = async (token: string) => {
try {
@@ -28,27 +29,58 @@ export default defineNuxtPlugin(async (nuxtApp) => {
await init(nuxtApp.$state.token.value)
}
+ const send = (name: string, data: any) => {
+ if (socket) {
+ const _id = messageIndex++
+ socket.emit(name, { _id, data })
+ return _id
+ }
+ }
+
const jobs = {
- subscribe(name: string, id: string, cb: (data: any) => void) {
- if (socket) {
- socket.emit('subscribe', { name, id })
- const tempFn = (data: any) => {
- if (data.id === id && data.name === name) {
- cb(data)
- if (data.status === 'completed' || data.status === 'failed') {
- socket?.off('status', tempFn)
- }
+ subscribe(
+ job: { id: string; name: string } | any,
+ subscribedCb?: () => void,
+ statusCb?: (status: 'active' | 'completed' | 'failed' | 'refresh', error?: any) => void,
+ logCb?: (data: { message: string }) => void,
+ ) {
+ const logFn = (data: { id: string; name: string; data: { message: string } }) => {
+ if (data.id === job.id) {
+ if (logCb) logCb(data.data)
+ }
+ }
+ const statusFn = (data: any) => {
+ if (data.id === job.id) {
+ if (statusCb) statusCb(data.status, data.error)
+ if (data.status === 'completed' || data.status === 'failed') {
+ socket?.off('status', statusFn)
+ socket?.off('log', logFn)
+ }
+ }
+ }
+
+ const _id = send('subscribe', job)
+
+ const subscribeFn = (data: { _id: number; name: string; id: string }) => {
+ if (data._id === _id) {
+ if (data.id !== job.id || data.name !== job.name) {
+ job.id = data.id
+ job.name = data.name
}
+ if (subscribedCb) subscribedCb()
+ socket?.on('log', logFn)
+ socket?.on('status', statusFn)
+ socket?.off('subscribed', subscribeFn)
}
- socket.on('status', tempFn)
}
+ socket?.on('subscribed', subscribeFn)
},
getStatus(name: string, id: string): Promise
{
return new Promise((resolve) => {
if (socket) {
- socket.emit('status', { name, id })
+ const _id = send('status', { name, id })
const tempFn = (data: any) => {
- if (data.id === id && data.name === name) {
+ if (data._id === _id) {
resolve(data.status)
socket?.off('status', tempFn)
}
diff --git a/packages/nocodb-nest/src/controllers/imports/helpers/readAndProcessData.ts b/packages/nocodb-nest/src/controllers/imports/helpers/readAndProcessData.ts
index 45cbc3bd2e..7de16fad7e 100644
--- a/packages/nocodb-nest/src/controllers/imports/helpers/readAndProcessData.ts
+++ b/packages/nocodb-nest/src/controllers/imports/helpers/readAndProcessData.ts
@@ -1,7 +1,8 @@
+/* eslint-disable no-async-promise-executor */
import { RelationTypes, UITypes } from 'nocodb-sdk';
import EntityMap from './EntityMap';
-import type { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
-import type { TablesService } from '../../../services/tables.service';
+import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
+import type { TablesService } from '../../../../services/tables.service';
// @ts-ignore
import type { AirtableBase } from 'airtable/lib/airtable_base';
import type { TableType } from 'nocodb-sdk';
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/at-import.controller.ts b/packages/nocodb-nest/src/modules/jobs/at-import/at-import.controller.ts
new file mode 100644
index 0000000000..51ec6657ff
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/at-import.controller.ts
@@ -0,0 +1,76 @@
+import { InjectQueue } from '@nestjs/bull';
+import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common';
+import { GlobalGuard } from 'src/guards/global/global.guard';
+import { ExtractProjectIdMiddleware } from 'src/middlewares/extract-project-id/extract-project-id.middleware';
+import { Queue } from 'bull';
+import { SyncSource } from 'src/models';
+import { NcError } from 'src/helpers/catchError';
+import { QueueService } from '../fallback-queue.service';
+import { JobsService } from '../jobs.service';
+
+@Controller()
+@UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
+export class AtImportController {
+ activeQueue;
+ constructor(
+ @InjectQueue('jobs') private readonly jobsQueue: Queue,
+ private readonly fallbackQueueService: QueueService,
+ private readonly jobsService: JobsService,
+ ) {
+ this.activeQueue = process.env.NC_REDIS_URL
+ ? this.jobsQueue
+ : this.fallbackQueueService;
+ }
+
+ @Post('/api/v1/db/meta/import/airtable')
+ @HttpCode(200)
+ async importAirtable(@Request() req) {
+ const job = await this.activeQueue.add('at-import', {
+ ...req.body,
+ });
+
+ return { id: job.id, name: job.name };
+ }
+
+ @Post('/api/v1/db/meta/syncs/:syncId/trigger')
+ @HttpCode(200)
+ async triggerSync(@Request() req) {
+ const jobs = await this.jobsService.jobList('at-import');
+ const fnd = jobs.find((j) => j.data.syncId === req.params.syncId);
+
+ if (fnd) {
+ NcError.badRequest('Sync already in progress');
+ }
+
+ const syncSource = await SyncSource.get(req.params.syncId);
+
+ const user = await syncSource.getUser();
+
+ // Treat default baseUrl as siteUrl from req object
+ let baseURL = (req as any).ncSiteUrl;
+
+ // if environment value avail use it
+ // or if it's docker construct using `PORT`
+ if (process.env.NC_DOCKER) {
+ baseURL = `http://localhost:${process.env.PORT || 8080}`;
+ }
+
+ const job = await this.activeQueue.add('at-import', {
+ syncId: req.params.syncId,
+ ...(syncSource?.details || {}),
+ projectId: syncSource.project_id,
+ baseId: syncSource.base_id,
+ authToken: '',
+ baseURL,
+ user: user,
+ });
+
+ return { id: job.id, name: job.name };
+ }
+
+ @Post('/api/v1/db/meta/syncs/:syncId/abort')
+ @HttpCode(200)
+ async abortImport(@Request() req) {
+ return {};
+ }
+}
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/at-import.processor.ts b/packages/nocodb-nest/src/modules/jobs/at-import/at-import.processor.ts
new file mode 100644
index 0000000000..0ab6e7e30c
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/at-import.processor.ts
@@ -0,0 +1,2515 @@
+import { promisify } from 'util';
+import { UITypes } from 'nocodb-sdk';
+import Airtable from 'airtable';
+import jsonfile from 'jsonfile';
+import hash from 'object-hash';
+import { T } from 'nc-help';
+import dayjs from 'dayjs';
+import utc from 'dayjs/plugin/utc';
+import tinycolor from 'tinycolor2';
+import { Process, Processor } from '@nestjs/bull';
+import { Job } from 'bull';
+import extractRolesObj from '../../../utils/extractRolesObj';
+import { AttachmentsService } from '../../../services/attachments.service';
+import { ColumnsService } from '../../../services/columns.service';
+import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
+import { FiltersService } from '../../../services/filters.service';
+import { FormColumnsService } from '../../../services/form-columns.service';
+import { GalleriesService } from '../../../services/galleries.service';
+import { GridsService } from '../../../services/grids.service';
+import { ProjectUsersService } from '../../../services/project-users/project-users.service';
+import { ProjectsService } from '../../../services/projects.service';
+import { SortsService } from '../../../services/sorts.service';
+import { TablesService } from '../../../services/tables.service';
+import { ViewColumnsService } from '../../../services/view-columns.service';
+import { ViewsService } from '../../../services/views.service';
+import { FormsService } from '../../../services/forms.service';
+import { JobsEventService } from '../jobs-event.service';
+import FetchAT from './helpers/fetchAT';
+import { importData, importLTARData } from './helpers/readAndProcessData';
+import EntityMap from './helpers/EntityMap';
+import type { UserType } from 'nocodb-sdk';
+
+const writeJsonFileAsync = promisify(jsonfile.writeFile);
+
+dayjs.extend(utc);
+
+const selectColors = {
+ // normal
+ blue: '#cfdfff',
+ cyan: '#d0f0fd',
+ teal: '#c2f5e9',
+ green: '#d1f7c4',
+ orange: '#fee2d5',
+ yellow: '#ffeab6',
+ red: '#ffdce5',
+ pink: '#ffdaf6',
+ purple: '#ede2fe',
+ gray: '#eee',
+ // medium
+ blueMedium: '#9cc7ff',
+ cyanMedium: '#77d1f3',
+ tealMedium: '#72ddc3',
+ greenMedium: '#93e088',
+ orangeMedium: '#ffa981',
+ yellowMedium: '#ffd66e',
+ redMedium: '#ff9eb7',
+ pinkMedium: '#f99de2',
+ purpleMedium: '#cdb0ff',
+ grayMedium: '#ccc',
+ // dark
+ blueDark: '#2d7ff9',
+ cyanDark: '#18bfff',
+ tealDark: '#20d9d2',
+ greenDark: '#20c933',
+ orangeDark: '#ff6f2c',
+ yellowDark: '#fcb400',
+ redDark: '#f82b60',
+ pinkDark: '#ff08c2',
+ purpleDark: '#8b46ff',
+ grayDark: '#666',
+ // darker
+ blueDarker: '#2750ae',
+ cyanDarker: '#0b76b7',
+ tealDarker: '#06a09b',
+ greenDarker: '#338a17',
+ orangeDarker: '#d74d26',
+ yellowDarker: '#b87503',
+ redDarker: '#ba1e45',
+ pinkDarker: '#b2158b',
+ purpleDarker: '#6b1cb0',
+ grayDarker: '#444',
+};
+
+@Processor('jobs')
+export class AtImportProcessor {
+ constructor(
+ private readonly tablesService: TablesService,
+ private readonly viewsService: ViewsService,
+ private readonly projectsService: ProjectsService,
+ private readonly attachmentsService: AttachmentsService,
+ private readonly columnsService: ColumnsService,
+ private readonly filtersService: FiltersService,
+ private readonly formColumnsService: FormColumnsService,
+ private readonly formsService: FormsService,
+ private readonly galleriesService: GalleriesService,
+ private readonly gridsService: GridsService,
+ private readonly projectUsersService: ProjectUsersService,
+ private readonly viewColumnsService: ViewColumnsService,
+ private readonly sortsService: SortsService,
+ private readonly bulkDataAliasService: BulkDataAliasService,
+ private readonly jobsEventService: JobsEventService,
+ ) {}
+
+ @Process('at-import')
+ async job(job: Job) {
+ const syncDB = job.data;
+
+ const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent');
+ await sMapEM.init();
+ const userRole = syncDB.user.roles
+ .split(',')
+ .reduce((rolesObj, role) => ({ [role]: true, ...rolesObj }), {});
+
+ const sMap = {
+ // static mapping records between aTblId && ncId
+ async addToMappingTbl(aTblId, ncId, ncName, ncParent?) {
+ await sMapEM.addRow({ aTblId, ncId, ncName, ncParent });
+ },
+
+ // get NcID from airtable ID
+ async getNcIdFromAtId(aId) {
+ return (await sMapEM.getRow('aTblId', aId, ['ncId']))?.ncId;
+ },
+
+ // get nc Parent from airtable ID
+ async getNcParentFromAtId(aId) {
+ return (await sMapEM.getRow('aTblId', aId, ['ncParent']))?.ncParent;
+ },
+
+ // get nc-title from airtable ID
+ async getNcNameFromAtId(aId) {
+ return (await sMapEM.getRow('aTblId', aId, ['ncName']))?.ncName;
+ },
+ };
+
+ const logBasic = (log) => {
+ this.jobsEventService.sendLog(job, { message: log });
+ };
+
+ const logDetailed = (log) => {
+ if (debugMode) this.jobsEventService.sendLog(job, { message: log });
+ };
+
+ const perfStats = [];
+
+ const recordPerfStart = () => {
+ if (!debugMode) return 0;
+ return Date.now();
+ };
+
+ const recordPerfStats = (start, event) => {
+ if (!debugMode) return;
+ const duration = Date.now() - start;
+ perfStats.push({ d: duration, e: event });
+ };
+
+ let base, baseId;
+ const start = Date.now();
+ const enableErrorLogs = false;
+ const generate_migrationStats = true;
+ const debugMode = false;
+ let g_aTblSchema = [];
+ let ncCreatedProjectSchema: any = {};
+ const ncLinkMappingTable: any[] = [];
+ const nestedLookupTbl: any[] = [];
+ const nestedRollupTbl: any[] = [];
+ const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' };
+ const storeLinks = false;
+ const ncLinkDataStore: any = {};
+ const insertedAssocRef: any = {};
+
+ const atNcAliasRef: {
+ [ncTableId: string]: {
+ [ncTitle: string]: string;
+ };
+ } = {};
+
+ const uniqueTableNameGen = getUniqueNameGenerator('sheet');
+
+ // run time counter (statistics)
+ const rtc = {
+ sort: 0,
+ filter: 0,
+ view: {
+ total: 0,
+ grid: 0,
+ gallery: 0,
+ form: 0,
+ },
+ fetchAt: {
+ count: 0,
+ time: 0,
+ },
+ migrationSkipLog: {
+ count: 0,
+ log: [],
+ },
+ data: {
+ records: 0,
+ nestedLinks: 0,
+ },
+ };
+
+ const updateMigrationSkipLog = (tbl, col, type, reason?) => {
+ rtc.migrationSkipLog.count++;
+ rtc.migrationSkipLog.log.push(
+ `tn[${tbl}] cn[${col}] type[${type}] :: ${reason}`,
+ );
+ };
+
+ // mapping table
+ //
+
+ const getAirtableSchema = async (sDB) => {
+ const start = Date.now();
+
+ if (!sDB.shareId)
+ throw {
+ message:
+ 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details',
+ };
+
+ if (sDB.shareId.startsWith('exp')) {
+ const template = await FetchAT.readTemplate(sDB.shareId);
+ await FetchAT.initialize(template.template.exploreApplication.shareId);
+ } else {
+ await FetchAT.initialize(sDB.shareId);
+ }
+ const ft = await FetchAT.read();
+ const duration = Date.now() - start;
+ rtc.fetchAt.count++;
+ rtc.fetchAt.time += duration;
+
+ if (!ft.baseId) {
+ throw {
+ message:
+ 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details',
+ };
+ }
+
+ const file = ft.schema;
+ baseId = ft.baseId;
+ base = new Airtable({ apiKey: sDB.apiKey }).base(baseId);
+ // store copy of airtable schema globally
+ g_aTblSchema = file.tableSchemas;
+
+ if (debugMode)
+ await writeJsonFileAsync('aTblSchema.json', ft, { spaces: 2 });
+
+ return file;
+ };
+
+ const getViewData = async (viewId) => {
+ const start = Date.now();
+ const ft = await FetchAT.readView(viewId);
+ const duration = Date.now() - start;
+ rtc.fetchAt.count++;
+ rtc.fetchAt.time += duration;
+
+ if (debugMode)
+ await writeJsonFileAsync(`${viewId}.json`, ft, { spaces: 2 });
+ return ft.view;
+ };
+
+ const getRootDbType = () => {
+ return ncCreatedProjectSchema?.bases.find((el) => el.id === syncDB.baseId)
+ ?.type;
+ };
+
+ // base mapping table
+ const aTblNcTypeMap = {
+ foreignKey: UITypes.LinkToAnotherRecord,
+ text: UITypes.SingleLineText,
+ multilineText: UITypes.LongText,
+ richText: UITypes.LongText,
+ multipleAttachment: UITypes.Attachment,
+ checkbox: UITypes.Checkbox,
+ multiSelect: UITypes.MultiSelect,
+ select: UITypes.SingleSelect,
+ collaborator: UITypes.Collaborator,
+ multiCollaborator: UITypes.Collaborator,
+ date: UITypes.Date,
+ phone: UITypes.PhoneNumber,
+ number: UITypes.Decimal,
+ rating: UITypes.Rating,
+ formula: UITypes.Formula,
+ rollup: UITypes.Rollup,
+ count: UITypes.Count,
+ lookup: UITypes.Lookup,
+ autoNumber: UITypes.AutoNumber,
+ barcode: UITypes.SingleLineText,
+ button: UITypes.Button,
+ };
+
+ //-----------------------------------------------------------------------------
+ // aTbl helper routines
+ //
+
+ const nc_sanitizeName = (name) => {
+ // replace all special characters by _
+ return name.replace(/\W+/g, '_').trim();
+ };
+
+ const nc_getSanitizedColumnName = (table, name) => {
+ let col_name = nc_sanitizeName(name);
+
+ // truncate to 60 chars if character if exceeds above 60
+ col_name = col_name?.slice(0, 60);
+
+ // for knex, replace . with _
+ const col_alias = name.trim().replace(/\./g, '_');
+
+ // check if already a column exists with same name?
+ const duplicateTitle = table.columns.find(
+ (x) => x.title?.toLowerCase() === col_alias?.toLowerCase(),
+ );
+ const duplicateColumn = table.columns.find(
+ (x) => x.column_name?.toLowerCase() === col_name?.toLowerCase(),
+ );
+ if (duplicateTitle) {
+ if (enableErrorLogs) console.log(`## Duplicate title ${col_alias}`);
+ }
+
+ return {
+ // kludge: error observed in Nc with space around column-name
+ title: col_alias + (duplicateTitle ? '_2' : ''),
+ column_name: col_name + (duplicateColumn ? '_2' : ''),
+ };
+ };
+
+ const ncSchema = {
+ tables: [],
+ tablesById: {},
+ };
+
+ // aTbl: retrieve column name from column ID
+ //
+ const aTbl_getColumnName = (colId): any => {
+ for (let i = 0; i < g_aTblSchema.length; i++) {
+ const sheetObj = g_aTblSchema[i];
+ const column = sheetObj.columns.find((col) => col.id === colId);
+ if (column !== undefined)
+ return {
+ tn: sheetObj.name,
+ cn: column.name,
+ };
+ }
+ };
+
+ const nc_DumpTableSchema = async () => {
+ console.log('[');
+ // const ncTblList = await api.base.tableList(
+ // ncCreatedProjectSchema.id,
+ // syncDB.baseId
+ // );
+
+ const ncTblList = { list: [] };
+ ncTblList['list'] = await this.tablesService.getAccessibleTables({
+ projectId: ncCreatedProjectSchema.id,
+ baseId: syncDB.baseId,
+ roles: userRole,
+ });
+
+ for (let i = 0; i < ncTblList.list.length; i++) {
+ // const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
+ const ncTbl = await this.tablesService.getTableWithAccessibleViews({
+ tableId: ncTblList.list[i].id,
+ user: syncDB.user,
+ });
+ console.log(JSON.stringify(ncTbl, null, 2));
+ console.log(',');
+ }
+ console.log(']');
+ };
+
+ // retrieve nc column schema from using aTbl field ID as reference
+ //
+ const nc_getColumnSchema = async (aTblFieldId) => {
+ const ncTblId = await sMap.getNcParentFromAtId(aTblFieldId);
+ const ncColId = await sMap.getNcIdFromAtId(aTblFieldId);
+
+ // not migrated column, skip
+ if (ncColId === undefined || ncTblId === undefined) return 0;
+
+ return ncSchema.tablesById[ncTblId].columns.find((x) => x.id === ncColId);
+ };
+
+ // retrieve nc table schema using table name
+ // optimize: create a look-up table & re-use information
+ //
+ const nc_getTableSchema = async (tableName) => {
+ return ncSchema.tables.find((x) => x.title === tableName);
+ };
+
+ // delete project if already exists
+ const init = async ({
+ projectName,
+ }: {
+ projectName?: string;
+ projectId?: string;
+ }) => {
+ // delete 'sample' project if already exists
+ const x = { list: [] };
+ x['list'] = await this.projectsService.projectList({
+ user: { id: syncDB.user.id, roles: extractRolesObj(syncDB.user.roles) },
+ });
+
+ const sampleProj = x.list.find((a) => a.title === projectName);
+ if (sampleProj) {
+ await this.projectsService.projectSoftDelete({
+ projectId: sampleProj.id,
+ });
+ }
+ logDetailed('Init');
+ };
+
+ // map UIDT
+ //
+ const getNocoType = (col) => {
+ // start with default map
+ let ncType = aTblNcTypeMap[col.type];
+
+ // types email & url are marked as text
+ // types currency & percent, duration are marked as number
+ // types createTime & modifiedTime are marked as formula
+
+ switch (col.type) {
+ case 'text':
+ if (col.typeOptions?.validatorName === 'email')
+ ncType = UITypes.Email;
+ else if (col.typeOptions?.validatorName === 'url')
+ ncType = UITypes.URL;
+ break;
+
+ case 'number':
+ // kludge: currency validation error with decimal places
+ if (col.typeOptions?.format === 'percentV2') ncType = UITypes.Percent;
+ else if (col.typeOptions?.format === 'duration')
+ ncType = UITypes.Duration;
+ else if (col.typeOptions?.format === 'currency')
+ ncType = UITypes.Currency;
+ else if (col.typeOptions?.precision > 0) ncType = UITypes.Decimal;
+ break;
+
+ case 'formula':
+ if (col.typeOptions?.formulaTextParsed === 'CREATED_TIME()')
+ ncType = UITypes.DateTime;
+ else if (
+ col.typeOptions?.formulaTextParsed === 'LAST_MODIFIED_TIME()'
+ )
+ ncType = UITypes.DateTime;
+ break;
+
+ case 'computation':
+ if (col.typeOptions?.resultType === 'collaborator')
+ ncType = UITypes.Collaborator;
+ break;
+
+ case 'date':
+ if (col.typeOptions?.isDateTime) ncType = UITypes.DateTime;
+ break;
+ }
+
+ return ncType;
+ };
+
+ // retrieve additional options associated with selected data types
+ //
+ const getNocoTypeOptions = async (col: any): Promise => {
+ switch (col.type) {
+ case 'select':
+ case 'multiSelect': {
+ // prepare options list in CSV format
+ // note: NC doesn't allow comma's in options
+ //
+ const options = [];
+ let order = 1;
+ for (const [, value] of Object.entries(col.typeOptions.choices)) {
+ // replace commas with dot for multiselect
+ if (col.type === 'multiSelect') {
+ (value as any).name = (value as any).name.replace(/,/g, '.');
+ }
+ // we don't allow empty records, placeholder instead
+ if ((value as any).name === '') {
+ (value as any).name = 'nc_empty';
+ }
+ // enumerate duplicates (we don't allow them)
+ // TODO fix record mapping (this causes every record to map first option,
+ // we can't handle them using data api as they don't provide option id
+ // within data we might instead get the correct mapping from schema file )
+ let dupNo = 1;
+ const defaultName = (value as any).name;
+ while (
+ options.find(
+ (el) =>
+ el.title.toLowerCase() === (value as any).name.toLowerCase(),
+ )
+ ) {
+ (value as any).name = `${defaultName}_${dupNo++}`;
+ }
+ options.push({
+ order: order++,
+ title: (value as any).name,
+ color: selectColors[(value as any).color]
+ ? selectColors[(value as any).color]
+ : tinycolor.random().toHexString(),
+ });
+
+ await sMap.addToMappingTbl(
+ (value as any).id,
+ undefined,
+ (value as any).name,
+ );
+ }
+ return { type: col.type, data: options };
+ }
+ default:
+ return { type: undefined };
+ }
+ };
+
+ // convert to Nc schema (basic, excluding relations)
+ //
+ const tablesPrepare = async (tblSchema: any[]) => {
+ const tables: any[] = [];
+
+ for (let i = 0; i < tblSchema.length; ++i) {
+ const table: any = {};
+
+ if (syncDB.options.syncViews) {
+ rtc.view.total += tblSchema[i].views.reduce(
+ (acc, cur) =>
+ ['grid', 'form', 'gallery'].includes(cur.type) ? ++acc : acc,
+ 0,
+ );
+ } else {
+ rtc.view.total = tblSchema.length;
+ }
+
+ // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id;
+ table.title = tblSchema[i].name;
+ let sanitizedName = nc_sanitizeName(tblSchema[i].name);
+
+ // truncate to 50 chars if character if exceeds above 50
+ // upto 64 should be fine but we are keeping it to 50 since
+ // meta project adds prefix as well
+ sanitizedName = sanitizedName?.slice(0, 50);
+
+ // check for duplicate and populate a unique name if already exist
+ table.table_name = uniqueTableNameGen(sanitizedName);
+
+ const uniqueColNameGen = getUniqueNameGenerator('field');
+ table.columns = [];
+ const sysColumns = [
+ {
+ title: ncSysFields.id,
+ column_name: ncSysFields.id,
+ uidt: UITypes.ID,
+ meta: {
+ ag: 'nc',
+ },
+ },
+ {
+ title: ncSysFields.hash,
+ column_name: ncSysFields.hash,
+ uidt: UITypes.SingleLineText,
+ system: true,
+ },
+ ];
+
+ for (let j = 0; j < tblSchema[i].columns.length; j++) {
+ const col = tblSchema[i].columns[j];
+
+ // skip link, lookup, rollup fields in this iteration
+ if (['foreignKey', 'lookup', 'rollup'].includes(col.type)) {
+ continue;
+ }
+
+ // base column schema
+ const ncName: any = nc_getSanitizedColumnName(table, col.name);
+ const ncCol: any = {
+ // Enable to use aTbl identifiers as is: id: col.id,
+ title: ncName.title,
+ column_name: uniqueColNameGen(ncName.column_name),
+ uidt: getNocoType(col),
+ };
+
+ // not supported datatype: pure formula field
+ // allow formula based computed fields (created time/ modified time to go through)
+ if (ncCol.uidt === UITypes.Formula) {
+ updateMigrationSkipLog(
+ tblSchema[i].name,
+ ncName.title,
+ col.type,
+ 'column type not supported',
+ );
+ continue;
+ }
+
+ // change from default 'tinytext' as airtable allows more than 255 characters
+ // for single line text column type
+ if (col.type === 'text') ncCol.dt = 'text';
+
+ // #fix-2363-decimal-out-of-range
+ if (['sqlite3', 'mysql2'].includes(getRootDbType())) {
+ if (ncCol.uidt === UITypes.Decimal) {
+ ncCol.dt = 'double';
+ ncCol.dtxp = 22;
+ ncCol.dtxs = '2';
+ }
+ }
+
+ // additional column parameters when applicable
+ const colOptions = await getNocoTypeOptions(col);
+
+ switch (colOptions.type) {
+ case 'select':
+ case 'multiSelect':
+ ncCol.colOptions = {
+ options: [...colOptions.data],
+ };
+
+ if (['mysql', 'mysql2'].includes(getRootDbType())) {
+ // if options are empty, configure '' as an option
+ ncCol.dtxp =
+ colOptions.data
+ .map((el) => `'${el.title.replace(/'/gi, "''")}'`)
+ .join(',') || "''";
+ }
+
+ break;
+ case undefined:
+ break;
+ }
+ table.columns.push(ncCol);
+ }
+ table.columns.push(sysColumns[0]);
+ table.columns.push(sysColumns[1]);
+
+ tables.push(table);
+ }
+ return tables;
+ };
+
+ const nocoCreateBaseSchema = async (aTblSchema) => {
+ // base schema preparation: exclude
+ const tables: any[] = await tablesPrepare(aTblSchema);
+
+ // for each table schema, create nc table
+ for (let idx = 0; idx < tables.length; idx++) {
+ logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`);
+
+ logDetailed(`NC API: base.tableCreate ${tables[idx].title}`);
+
+ let _perfStart = recordPerfStart();
+ const table = await this.tablesService.tableCreate({
+ baseId: syncDB.baseId,
+ projectId: ncCreatedProjectSchema.id,
+ table: tables[idx],
+ user: syncDB.user,
+ });
+ recordPerfStats(_perfStart, 'dbTable.create');
+
+ updateNcTblSchema(table);
+
+ // update mapping table
+ await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title);
+ for (let colIdx = 0; colIdx < table.columns.length; colIdx++) {
+ const aId = aTblSchema[idx].columns.find(
+ (x) =>
+ x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title,
+ )?.id;
+ if (aId)
+ await sMap.addToMappingTbl(
+ aId,
+ table.columns[colIdx].id,
+ table.columns[colIdx].title,
+ table.id,
+ );
+ }
+
+ // update default view name- to match it to airtable view name
+ logDetailed(`NC API: dbView.list ${table.id}`);
+ _perfStart = recordPerfStart();
+ const view = { list: [] };
+ view['list'] = await this.viewsService.viewList({
+ tableId: table.id,
+ user: { roles: userRole },
+ });
+ recordPerfStats(_perfStart, 'dbView.list');
+
+ const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid');
+ logDetailed(
+ `NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`,
+ );
+ _perfStart = recordPerfStart();
+ await this.viewsService.viewUpdate({
+ viewId: view.list[0].id,
+ view: { title: aTbl_grid.name },
+ });
+ recordPerfStats(_perfStart, 'dbView.update');
+
+ await updateNcTblSchemaById(table.id);
+
+ await sMap.addToMappingTbl(
+ aTbl_grid.id,
+ table.views[0].id,
+ aTbl_grid.name,
+ table.id,
+ );
+ }
+
+ return tables;
+ };
+
+ const nocoCreateLinkToAnotherRecord = async (aTblSchema) => {
+ // Link to another RECORD
+ for (let idx = 0; idx < aTblSchema.length; idx++) {
+ const aTblLinkColumns = aTblSchema[idx].columns.filter(
+ (x) => x.type === 'foreignKey',
+ );
+
+ // Link columns exist
+ //
+ if (aTblLinkColumns.length) {
+ for (let i = 0; i < aTblLinkColumns.length; i++) {
+ logDetailed(
+ `[${idx + 1}/${aTblSchema.length}] Configuring Links :: [${
+ i + 1
+ }/${aTblLinkColumns.length}] ${aTblSchema[idx].name}`,
+ );
+
+ // for self links, there is no symmetric column
+ {
+ const src = aTbl_getColumnName(aTblLinkColumns[i].id);
+ const dst = aTbl_getColumnName(
+ aTblLinkColumns[i].typeOptions?.symmetricColumnId,
+ );
+ logDetailed(
+ `LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst?.tn}:${dst?.cn}`,
+ );
+ }
+
+ // check if link already established?
+ if (!nc_isLinkExists(aTblLinkColumns[i].id)) {
+ // parent table ID
+ const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+
+ // find child table name from symmetric column ID specified
+ // self link, symmetricColumnId field will be undefined
+ const childTable = aTbl_getColumnName(
+ aTblLinkColumns[i].typeOptions?.symmetricColumnId,
+ );
+
+ // retrieve child table ID (nc) from table name
+ let childTableId = srcTableId;
+ if (childTable) {
+ childTableId = (await nc_getTableSchema(childTable.tn)).id;
+ }
+
+ // check if already a column exists with this name?
+ let _perfStart = recordPerfStart();
+ const srcTbl: any =
+ await this.tablesService.getTableWithAccessibleViews({
+ tableId: srcTableId,
+ user: syncDB.user,
+ });
+ recordPerfStats(_perfStart, 'dbTable.read');
+
+ // create link
+ const ncName = nc_getSanitizedColumnName(
+ srcTbl,
+ aTblLinkColumns[i].name,
+ );
+
+ // LTAR alias ref to AT
+ atNcAliasRef[srcTbl.id] = atNcAliasRef[srcTbl.id] || {};
+ atNcAliasRef[srcTbl.id][ncName.title] = aTblLinkColumns[i].name;
+
+ logDetailed(
+ `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}`,
+ );
+ _perfStart = recordPerfStart();
+ const ncTbl: any = await this.columnsService.columnAdd({
+ tableId: srcTableId,
+ column: {
+ uidt: UITypes.LinkToAnotherRecord,
+ title: ncName.title,
+ column_name: ncName.column_name,
+ parentId: srcTableId,
+ childId: childTableId,
+ type: 'mm',
+ },
+ req: {
+ user: syncDB.user.email,
+ clientIp: '',
+ },
+ });
+ recordPerfStats(_perfStart, 'dbTableColumn.create');
+
+ updateNcTblSchema(ncTbl);
+
+ const ncId = ncTbl.columns.find(
+ (x) => x.title === ncName.title,
+ )?.id;
+ await sMap.addToMappingTbl(
+ aTblLinkColumns[i].id,
+ ncId,
+ ncName.title,
+ ncTbl.id,
+ );
+
+ // store link information in separate table
+ // this information will be helpful in identifying relation pair
+ const link = {
+ nc: {
+ title: ncName.title,
+ parentId: srcTableId,
+ childId: childTableId,
+ type: 'mm',
+ },
+ aTbl: {
+ tblId: aTblSchema[idx].id,
+ ...aTblLinkColumns[i],
+ },
+ };
+
+ ncLinkMappingTable.push(link);
+ } else {
+ // if link already exists, we need to change name of linked column
+ // to what is represented in airtable
+
+ // 1. extract associated link information from link table
+ // 2. retrieve parent table information (source)
+ // 3. using foreign parent & child column ID, find associated mapping in child table
+ // 4. update column name
+ const x = ncLinkMappingTable.findIndex(
+ (x) =>
+ x.aTbl.tblId ===
+ aTblLinkColumns[i].typeOptions.foreignTableId &&
+ x.aTbl.id ===
+ aTblLinkColumns[i].typeOptions.symmetricColumnId,
+ );
+
+ let _perfStart = recordPerfStart();
+ const childTblSchema: any =
+ await this.tablesService.getTableWithAccessibleViews({
+ tableId: ncLinkMappingTable[x].nc.childId,
+ user: syncDB.user,
+ });
+ recordPerfStats(_perfStart, 'dbTable.read');
+
+ _perfStart = recordPerfStart();
+ const parentTblSchema: any =
+ await this.tablesService.getTableWithAccessibleViews({
+ tableId: ncLinkMappingTable[x].nc.parentId,
+ user: syncDB.user,
+ });
+ recordPerfStats(_perfStart, 'dbTable.read');
+
+ let parentLinkColumn = parentTblSchema.columns.find(
+ (col) => col.title === ncLinkMappingTable[x].nc.title,
+ );
+
+ if (parentLinkColumn === undefined) {
+ updateMigrationSkipLog(
+ parentTblSchema?.title,
+ ncLinkMappingTable[x].nc.title,
+ UITypes.LinkToAnotherRecord,
+ 'Link error',
+ );
+ continue;
+ }
+
+ // hack // fix me
+ if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') {
+ parentLinkColumn = parentTblSchema.columns.find(
+ (col) => col.title === ncLinkMappingTable[x].nc.title + '_2',
+ );
+ }
+
+ let childLinkColumn: any = {};
+
+ if (parentLinkColumn.colOptions.type == 'hm') {
+ // for hm:
+ // mapping between child & parent column id is direct
+ //
+ childLinkColumn = childTblSchema.columns.find(
+ (col) =>
+ col.uidt === UITypes.LinkToAnotherRecord &&
+ col.colOptions.fk_child_column_id ===
+ parentLinkColumn.colOptions.fk_child_column_id &&
+ col.colOptions.fk_parent_column_id ===
+ parentLinkColumn.colOptions.fk_parent_column_id,
+ );
+ } else {
+ // for mm:
+ // mapping between child & parent column id is inverted
+ //
+ childLinkColumn = childTblSchema.columns.find(
+ (col) =>
+ col.uidt === UITypes.LinkToAnotherRecord &&
+ col.colOptions.fk_child_column_id ===
+ parentLinkColumn.colOptions.fk_parent_column_id &&
+ col.colOptions.fk_parent_column_id ===
+ parentLinkColumn.colOptions.fk_child_column_id &&
+ col.colOptions.fk_mm_model_id ===
+ parentLinkColumn.colOptions.fk_mm_model_id,
+ );
+ }
+
+ // check if already a column exists with this name?
+ const duplicate = childTblSchema.columns.find(
+ (x) => x.title === aTblLinkColumns[i].name,
+ );
+ const suffix = duplicate ? '_2' : '';
+ if (duplicate)
+ if (enableErrorLogs)
+ console.log(`## Duplicate ${aTblLinkColumns[i].name}`);
+
+ // rename
+ // note that: current rename API requires us to send all parameters,
+ // not just title being renamed
+ const ncName = nc_getSanitizedColumnName(
+ childTblSchema,
+ aTblLinkColumns[i].name,
+ );
+
+ logDetailed(
+ `NC API: dbTableColumn.update rename symmetric column ${ncName.title}`,
+ );
+ _perfStart = recordPerfStart();
+ const ncTbl: any = await this.columnsService.columnUpdate({
+ columnId: childLinkColumn.id,
+ column: {
+ ...childLinkColumn,
+ title: ncName.title,
+ column_name: ncName.column_name,
+ },
+ });
+ recordPerfStats(_perfStart, 'dbTableColumn.update');
+
+ updateNcTblSchema(ncTbl);
+
+ const ncId = ncTbl.columns.find(
+ (x) => x.title === aTblLinkColumns[i].name + suffix,
+ )?.id;
+ await sMap.addToMappingTbl(
+ aTblLinkColumns[i].id,
+ ncId,
+ aTblLinkColumns[i].name + suffix,
+ ncTbl.id,
+ );
+ }
+ }
+ }
+ }
+ };
+
+ const nocoCreateLookups = async (aTblSchema) => {
+ // LookUps
+ for (let idx = 0; idx < aTblSchema.length; idx++) {
+ const aTblColumns = aTblSchema[idx].columns.filter(
+ (x) => x.type === 'lookup',
+ );
+
+ // parent table ID
+ const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+ const srcTableSchema = ncSchema.tablesById[srcTableId];
+
+ if (aTblColumns.length) {
+ // Lookup
+ for (let i = 0; i < aTblColumns.length; i++) {
+ logDetailed(
+ `[${idx + 1}/${aTblSchema.length}] Configuring Lookup :: [${
+ i + 1
+ }/${aTblColumns.length}] ${aTblSchema[idx].name}`,
+ );
+
+ // something is not right, skip
+ if (
+ aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds
+ ?.length
+ ) {
+ if (enableErrorLogs)
+ console.log(`## Invalid column IDs mapped; skip`);
+
+ updateMigrationSkipLog(
+ srcTableSchema.title,
+ aTblColumns[i].name,
+ aTblColumns[i].type,
+ 'invalid column ID in dependency list',
+ );
+ continue;
+ }
+
+ const ncRelationColumnId = await sMap.getNcIdFromAtId(
+ aTblColumns[i].typeOptions.relationColumnId,
+ );
+ const ncLookupColumnId = await sMap.getNcIdFromAtId(
+ aTblColumns[i].typeOptions.foreignTableRollupColumnId,
+ );
+
+ if (
+ ncLookupColumnId === undefined ||
+ ncRelationColumnId === undefined
+ ) {
+ aTblColumns[i]['srcTableId'] = srcTableId;
+ nestedLookupTbl.push(aTblColumns[i]);
+ continue;
+ }
+
+ const ncName = nc_getSanitizedColumnName(
+ srcTableSchema,
+ aTblColumns[i].name,
+ );
+
+ logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
+ const _perfStart = recordPerfStart();
+ const ncTbl: any = await this.columnsService.columnAdd({
+ tableId: srcTableId,
+ column: {
+ uidt: UITypes.Lookup,
+ title: ncName.title,
+ column_name: ncName.column_name,
+ fk_relation_column_id: ncRelationColumnId,
+ fk_lookup_column_id: ncLookupColumnId,
+ },
+ req: {
+ user: syncDB.user.email,
+ clientIp: '',
+ },
+ });
+ recordPerfStats(_perfStart, 'dbTableColumn.create');
+
+ updateNcTblSchema(ncTbl);
+
+ const ncId = ncTbl.columns.find(
+ (x) => x.title === aTblColumns[i].name,
+ )?.id;
+ await sMap.addToMappingTbl(
+ aTblColumns[i].id,
+ ncId,
+ aTblColumns[i].name,
+ ncTbl.id,
+ );
+ }
+ }
+ }
+
+ let level = 2;
+ let nestedCnt = 0;
+ while (nestedLookupTbl.length) {
+ // if nothing has changed from previous iteration, skip rest
+ if (nestedCnt === nestedLookupTbl.length) {
+ for (let i = 0; i < nestedLookupTbl.length; i++) {
+ const fTblField =
+ nestedLookupTbl[i].typeOptions.foreignTableRollupColumnId;
+ const name = aTbl_getColumnName(fTblField);
+ updateMigrationSkipLog(
+ ncSchema.tablesById[nestedLookupTbl[i].srcTableId]?.title,
+ nestedLookupTbl[i].name,
+ nestedLookupTbl[i].type,
+ `foreign table field not found [${name.tn}/${name.cn}]`,
+ );
+ }
+ if (enableErrorLogs)
+ console.log(
+ `## Failed to configure ${nestedLookupTbl.length} lookups`,
+ );
+ break;
+ }
+
+ // Nested lookup
+ nestedCnt = nestedLookupTbl.length;
+ for (let i = 0; i < nestedLookupTbl.length; i++) {
+ const srcTableId = nestedLookupTbl[0].srcTableId;
+ const srcTableSchema = ncSchema.tablesById[srcTableId];
+
+ const ncRelationColumnId = await sMap.getNcIdFromAtId(
+ nestedLookupTbl[0].typeOptions.relationColumnId,
+ );
+ const ncLookupColumnId = await sMap.getNcIdFromAtId(
+ nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId,
+ );
+
+ if (
+ ncLookupColumnId === undefined ||
+ ncRelationColumnId === undefined
+ ) {
+ continue;
+ }
+
+ const ncName = nc_getSanitizedColumnName(
+ srcTableSchema,
+ nestedLookupTbl[0].name,
+ );
+
+ logDetailed(
+ `Configuring Nested Lookup: Level-${level} [${i + 1}/${nestedCnt} ${
+ ncName.title
+ }]`,
+ );
+
+ logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
+ const _perfStart = recordPerfStart();
+ const ncTbl: any = await this.columnsService.columnAdd({
+ tableId: srcTableId,
+ column: {
+ uidt: UITypes.Lookup,
+ title: ncName.title,
+ column_name: ncName.column_name,
+ fk_relation_column_id: ncRelationColumnId,
+ fk_lookup_column_id: ncLookupColumnId,
+ },
+ req: {
+ user: syncDB.user.email,
+ clientIp: '',
+ },
+ });
+ recordPerfStats(_perfStart, 'dbTableColumn.create');
+
+ updateNcTblSchema(ncTbl);
+
+ const ncId = ncTbl.columns.find(
+ (x) => x.title === nestedLookupTbl[0].name,
+ )?.id;
+ await sMap.addToMappingTbl(
+ nestedLookupTbl[0].id,
+ ncId,
+ nestedLookupTbl[0].name,
+ ncTbl.id,
+ );
+
+ // remove entry
+ nestedLookupTbl.splice(0, 1);
+ }
+ level++;
+ }
+ };
+
+ const getRollupNcFunction = (aTblFunction) => {
+ const fn = aTblFunction.split('(')[0];
+ const aTbl_ncRollUp = {
+ AND: '',
+ ARRAYCOMPACT: '',
+ ARRAYJOIN: '',
+ ARRAYUNIQUE: '',
+ AVERAGE: 'average',
+ CONCATENATE: '',
+ COUNT: 'count',
+ COUNTA: '',
+ COUNTALL: '',
+ MAX: 'max',
+ MIN: 'min',
+ OR: '',
+ SUM: 'sum',
+ XOR: '',
+ };
+ return aTbl_ncRollUp[fn];
+ };
+
+ const nocoCreateRollup = async (aTblSchema) => {
+ // Rollup
+ for (let idx = 0; idx < aTblSchema.length; idx++) {
+ const aTblColumns = aTblSchema[idx].columns.filter(
+ (x) => x.type === 'rollup',
+ );
+
+ // parent table ID
+ const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+ const srcTableSchema = ncSchema.tablesById[srcTableId];
+
+ if (aTblColumns.length) {
+ // rollup exist
+ for (let i = 0; i < aTblColumns.length; i++) {
+ logDetailed(
+ `[${idx + 1}/${aTblSchema.length}] Configuring Rollup :: [${
+ i + 1
+ }/${aTblColumns.length}] ${aTblSchema[idx].name}`,
+ );
+
+ // fetch associated rollup function
+ // skip column creation if supported rollup function does not exist
+ const ncRollupFn = getRollupNcFunction(
+ aTblColumns[i].typeOptions.formulaTextParsed,
+ );
+
+ if (ncRollupFn === '' || ncRollupFn === undefined) {
+ updateMigrationSkipLog(
+ srcTableSchema.title,
+ aTblColumns[i].name,
+ aTblColumns[i].type,
+ `rollup function ${aTblColumns[i].typeOptions.formulaTextParsed} not supported`,
+ );
+ continue;
+ }
+
+ // something is not right, skip
+ if (
+ aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds
+ ?.length
+ ) {
+ if (enableErrorLogs)
+ console.log(`## Invalid column IDs mapped; skip`);
+
+ updateMigrationSkipLog(
+ srcTableSchema.title,
+ aTblColumns[i].name,
+ aTblColumns[i].type,
+ 'invalid column ID in dependency list',
+ );
+ continue;
+ }
+
+ const ncRelationColumnId = await sMap.getNcIdFromAtId(
+ aTblColumns[i].typeOptions.relationColumnId,
+ );
+ const ncRollupColumnId = await sMap.getNcIdFromAtId(
+ aTblColumns[i].typeOptions.foreignTableRollupColumnId,
+ );
+
+ if (ncRollupColumnId === undefined) {
+ aTblColumns[i]['srcTableId'] = srcTableId;
+ nestedRollupTbl.push(aTblColumns[i]);
+ continue;
+ }
+
+ // skip, if rollup column was pointing to another virtual column
+ const ncColSchema = await nc_getColumnSchema(
+ aTblColumns[i].typeOptions.foreignTableRollupColumnId,
+ );
+ if (
+ ncColSchema?.uidt === UITypes.Formula ||
+ ncColSchema?.uidt === UITypes.Lookup ||
+ ncColSchema?.uidt === UITypes.Rollup ||
+ ncColSchema?.uidt === UITypes.Checkbox
+ ) {
+ updateMigrationSkipLog(
+ srcTableSchema.title,
+ aTblColumns[i].name,
+ aTblColumns[i].type,
+ 'rollup referring to a column type not supported currently',
+ );
+ continue;
+ }
+
+ const ncName = nc_getSanitizedColumnName(
+ srcTableSchema,
+ aTblColumns[i].name,
+ );
+
+ logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`);
+ const _perfStart = recordPerfStart();
+ const ncTbl: any = await this.columnsService.columnAdd({
+ tableId: srcTableId,
+ column: {
+ uidt: UITypes.Rollup,
+ title: ncName.title,
+ column_name: ncName.column_name,
+ fk_relation_column_id: ncRelationColumnId,
+ fk_rollup_column_id: ncRollupColumnId,
+ rollup_function: ncRollupFn,
+ },
+ req: {
+ user: syncDB.user.email,
+ clientIp: '',
+ },
+ });
+ recordPerfStats(_perfStart, 'dbTableColumn.create');
+
+ updateNcTblSchema(ncTbl);
+
+ const ncId = ncTbl.columns.find(
+ (x) => x.title === aTblColumns[i].name,
+ )?.id;
+ await sMap.addToMappingTbl(
+ aTblColumns[i].id,
+ ncId,
+ aTblColumns[i].name,
+ ncTbl.id,
+ );
+ }
+ }
+ }
+ logDetailed(`Nested rollup: ${nestedRollupTbl.length}`);
+ };
+
+ const nocoLookupForRollup = async () => {
+ const nestedCnt = nestedLookupTbl.length;
+ for (let i = 0; i < nestedLookupTbl.length; i++) {
+ const srcTableId = nestedLookupTbl[0].srcTableId;
+ const srcTableSchema = ncSchema.tablesById[srcTableId];
+
+ const ncRelationColumnId = await sMap.getNcIdFromAtId(
+ nestedLookupTbl[0].typeOptions.relationColumnId,
+ );
+ const ncLookupColumnId = await sMap.getNcIdFromAtId(
+ nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId,
+ );
+
+ if (
+ ncLookupColumnId === undefined ||
+ ncRelationColumnId === undefined
+ ) {
+ continue;
+ }
+
+ const ncName = nc_getSanitizedColumnName(
+ srcTableSchema,
+ nestedLookupTbl[0].name,
+ );
+
+ logDetailed(
+ `Configuring Lookup over Rollup :: [${i + 1}/${nestedCnt}] ${
+ ncName.title
+ }`,
+ );
+
+ logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
+ const _perfStart = recordPerfStart();
+ const ncTbl: any = await this.columnsService.columnAdd({
+ tableId: srcTableId,
+ column: {
+ uidt: UITypes.Lookup,
+ title: ncName.title,
+ column_name: ncName.column_name,
+ fk_relation_column_id: ncRelationColumnId,
+ fk_lookup_column_id: ncLookupColumnId,
+ },
+ req: {
+ user: syncDB.user.email,
+ clientIp: '',
+ },
+ });
+ recordPerfStats(_perfStart, 'dbTableColumn.create');
+
+ updateNcTblSchema(ncTbl);
+
+ const ncId = ncTbl.columns.find(
+ (x) => x.title === nestedLookupTbl[0].name,
+ )?.id;
+ await sMap.addToMappingTbl(
+ nestedLookupTbl[0].id,
+ ncId,
+ nestedLookupTbl[0].name,
+ ncTbl.id,
+ );
+
+ // remove entry
+ nestedLookupTbl.splice(0, 1);
+ }
+ };
+
+ const nocoSetPrimary = async (aTblSchema) => {
+ for (let idx = 0; idx < aTblSchema.length; idx++) {
+ logDetailed(
+ `[${idx + 1}/${aTblSchema.length}] Configuring Display value : ${
+ aTblSchema[idx].name
+ }`,
+ );
+
+ const pColId = aTblSchema[idx].primaryColumnId;
+ const ncColId = await sMap.getNcIdFromAtId(pColId);
+
+ // skip primary column configuration if we field not migrated
+ if (ncColId) {
+ logDetailed(`NC API: dbTableColumn.primaryColumnSet`);
+ const _perfStart = recordPerfStart();
+ await this.columnsService.columnSetAsPrimary({ columnId: ncColId });
+ recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet');
+
+ // update schema
+ const ncTblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+ await updateNcTblSchemaById(ncTblId);
+ }
+ }
+ };
+
+ // retrieve nc-view column ID from corresponding nc-column ID
+ const nc_getViewColumnId = async (viewId, viewType, ncColumnId) => {
+ // retrieve view Info
+ let viewDetails;
+
+ const _perfStart = recordPerfStart();
+ if (viewType === 'form') {
+ viewDetails = (
+ await this.formsService.formViewGet({ formViewId: viewId })
+ ).columns;
+ recordPerfStats(_perfStart, 'dbView.formRead');
+ } else if (viewType === 'gallery') {
+ viewDetails = (
+ await this.galleriesService.galleryViewGet({ galleryViewId: viewId })
+ ).columns;
+ recordPerfStats(_perfStart, 'dbView.galleryRead');
+ } else {
+ viewDetails = await this.viewColumnsService.columnList({
+ viewId: viewId,
+ });
+ recordPerfStats(_perfStart, 'dbView.gridColumnsList');
+ }
+
+ return viewDetails.find((x) => x.fk_column_id === ncColumnId)?.id;
+ };
+
+ ////////// Data processing
+
+ const nocoBaseDataProcessing_v2 = async (sDB, table, record) => {
+ const recordHash = hash(record);
+ const rec = { ...record.fields };
+
+ // kludge -
+ // trim spaces on either side of column name
+ // leads to error in NocoDB
+ Object.keys(rec).forEach((key) => {
+ const replacedKey = key.trim().replace(/\./g, '_');
+ if (key !== replacedKey) {
+ rec[replacedKey] = rec[key];
+ delete rec[key];
+ }
+ });
+
+ // post-processing on the record
+ for (const [key, value] of Object.entries(
+ rec as { [key: string]: any },
+ )) {
+ // retrieve datatype
+ const dt = table.columns.find((x) => x.title === key)?.uidt;
+
+ // always process LTAR, Lookup, and Rollup columns as we delete the key after processing
+ if (
+ !value &&
+ dt !== UITypes.LinkToAnotherRecord &&
+ dt !== UITypes.Lookup &&
+ dt !== UITypes.Rollup
+ ) {
+ rec[key] = null;
+ continue;
+ }
+
+ switch (dt) {
+ // https://www.npmjs.com/package/validator
+ // default value: digits_after_decimal: [2]
+ // if currency, set decimal place to 2
+ //
+ case UITypes.Currency:
+ rec[key] = (+value).toFixed(2);
+ break;
+
+ // we will pick up LTAR once all table data's are in place
+ case UITypes.LinkToAnotherRecord:
+ if (storeLinks) {
+ if (ncLinkDataStore[table.title][record.id] === undefined)
+ ncLinkDataStore[table.title][record.id] = {
+ id: record.id,
+ fields: {},
+ };
+ ncLinkDataStore[table.title][record.id]['fields'][key] = value;
+ }
+ delete rec[key];
+ break;
+
+ // these will be automatically populated depending on schema configuration
+ case UITypes.Lookup:
+ case UITypes.Rollup:
+ delete rec[key];
+ break;
+
+ case UITypes.Collaborator:
+ // in case of multi-collaborator, this will be an array
+ if (Array.isArray(value)) {
+ let collaborators = '';
+ for (let i = 0; i < value.length; i++) {
+ collaborators += `${value[i]?.name} <${value[i]?.email}>, `;
+ rec[key] = collaborators;
+ }
+ } else rec[key] = `${value?.name} <${value?.email}>`;
+ break;
+
+ case UITypes.Button:
+ rec[key] = `${value?.label} <${value?.url}>`;
+ break;
+
+ case UITypes.DateTime:
+ case UITypes.CreateTime:
+ case UITypes.LastModifiedTime:
+ rec[key] = dayjs(value).format('YYYY-MM-DD HH:mm');
+ break;
+
+ case UITypes.Date:
+ if (/\d{5,}/.test(value)) {
+ // skip
+ rec[key] = null;
+ logBasic(`:: Invalid date ${value}`);
+ } else {
+ rec[key] = dayjs(value).format('YYYY-MM-DD');
+ }
+ break;
+
+ case UITypes.SingleSelect:
+ if (value === '') {
+ rec[key] = 'nc_empty';
+ }
+ rec[key] = value;
+ break;
+
+ case UITypes.MultiSelect:
+ rec[key] = value
+ ?.map((v) => {
+ if (v === '') {
+ return 'nc_empty';
+ }
+ return `${v.replace(/,/g, '.')}`;
+ })
+ .join(',');
+ break;
+
+ case UITypes.Attachment:
+ if (!syncDB.options.syncAttachment) rec[key] = null;
+ else {
+ let tempArr = [];
+
+ try {
+ logBasic(
+ ` :: Retrieving attachment :: ${value
+ ?.map((a) => a.filename?.split('?')?.[0])
+ .join(', ')}`,
+ );
+ tempArr = await this.attachmentsService.uploadViaURL({
+ path: `noco/${sDB.projectName}/${table.title}/${key}`,
+ urls: value?.map((attachment) => ({
+ fileName: attachment.filename?.split('?')?.[0],
+ url: attachment.url,
+ size: attachment.size,
+ mimetype: attachment.type,
+ })),
+ });
+ } catch (e) {
+ console.log(e);
+ }
+
+ rec[key] = JSON.stringify(tempArr);
+ }
+ break;
+
+ case UITypes.SingleLineText:
+ // Barcode data
+ if (value?.text) {
+ rec[key] = value.text;
+ }
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ // insert airtable record ID explicitly into each records
+ rec[ncSysFields.id] = record.id;
+ rec[ncSysFields.hash] = recordHash;
+
+ return rec;
+ };
+
+ const nocoReadDataSelected = async (projName, table, callback, fields) => {
+ return new Promise((resolve, reject) => {
+ base(table.title)
+ .select({
+ pageSize: 100,
+ // maxRecords: 100,
+ fields: fields,
+ })
+ .eachPage(
+ async function page(records, fetchNextPage) {
+ // This function (`page`) will get called for each page of records.
+ // records.forEach(record => callback(table, record));
+ logBasic(
+ `:: ${table.title} / ${fields} : ${
+ recordCnt + 1
+ } ~ ${(recordCnt += 100)}`,
+ );
+ await Promise.all(
+ records.map((r) => callback(projName, table, r, fields)),
+ );
+
+ // To fetch the next page of records, call `fetchNextPage`.
+ // If there are more records, `page` will get called again.
+ // If there are no more records, `done` will get called.
+ fetchNextPage();
+ },
+ function done(err) {
+ if (err) {
+ console.error(err);
+ reject(err);
+ }
+ resolve(null);
+ },
+ );
+ });
+ };
+
+ //////////
+
+ const nc_isLinkExists = (airtableFieldId) => {
+ return !!ncLinkMappingTable.find(
+ (x) => x.aTbl.typeOptions.symmetricColumnId === airtableFieldId,
+ );
+ };
+
+ const nocoCreateProject = async (projName) => {
+ // create empty project (XC-DB)
+ logDetailed(`Create Project: ${projName}`);
+ const _perfStart = recordPerfStart();
+
+ ncCreatedProjectSchema = await this.projectsService.projectCreate({
+ project: { title: projName },
+ user: { id: syncDB.user.id },
+ });
+
+ recordPerfStats(_perfStart, 'project.create');
+ };
+
+ const nocoGetProject = async (projId) => {
+ // create empty project (XC-DB)
+ logDetailed(`Getting project meta: ${projId}`);
+ const _perfStart = recordPerfStart();
+ ncCreatedProjectSchema = await this.projectsService.getProjectWithInfo({
+ projectId: projId,
+ });
+ recordPerfStats(_perfStart, 'project.read');
+ };
+
+ const nocoConfigureGalleryView = async (sDB, aTblSchema) => {
+ if (!sDB.options.syncViews) return;
+ for (let idx = 0; idx < aTblSchema.length; idx++) {
+ const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
+ const galleryViews = aTblSchema[idx].views.filter(
+ (x) => x.type === 'gallery',
+ );
+
+ const configuredViews =
+ rtc.view.grid + rtc.view.gallery + rtc.view.form;
+ rtc.view.gallery += galleryViews.length;
+
+ for (let i = 0; i < galleryViews.length; i++) {
+ logDetailed(` Axios fetch view-data`);
+
+ // create view
+ await getViewData(galleryViews[i].id);
+ const viewName = aTblSchema[idx].views.find(
+ (x) => x.id === galleryViews[i].id,
+ )?.name;
+
+ logBasic(
+ `:: [${configuredViews + i + 1}/${rtc.view.total}] Gallery : ${
+ aTblSchema[idx].name
+ } / ${viewName}`,
+ );
+
+ logDetailed(`NC API dbView.galleryCreate :: ${viewName}`);
+ const _perfStart = recordPerfStart();
+ await this.galleriesService.galleryViewCreate({
+ tableId: tblId,
+ gallery: {
+ title: viewName,
+ },
+ });
+ recordPerfStats(_perfStart, 'dbView.galleryCreate');
+
+ await updateNcTblSchemaById(tblId);
+ }
+ }
+ };
+
+ const nocoConfigureFormView = async (sDB, aTblSchema) => {
+ if (!sDB.options.syncViews) return;
+ for (let idx = 0; idx < aTblSchema.length; idx++) {
+ const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+ const formViews = aTblSchema[idx].views.filter(
+ (x) => x.type === 'form',
+ );
+
+ const configuredViews =
+ rtc.view.grid + rtc.view.gallery + rtc.view.form;
+ rtc.view.form += formViews.length;
+ for (let i = 0; i < formViews.length; i++) {
+ logDetailed(` Axios fetch view-data`);
+
+ // create view
+ const vData = await getViewData(formViews[i].id);
+ const viewName = aTblSchema[idx].views.find(
+ (x) => x.id === formViews[i].id,
+ )?.name;
+
+ logBasic(
+ `:: [${configuredViews + i + 1}/${rtc.view.total}] Form : ${
+ aTblSchema[idx].name
+ } / ${viewName}`,
+ );
+
+ // everything is default
+ let refreshMode = 'NO_REFRESH';
+ let msg = 'Thank you for submitting the form!';
+ let desc = '';
+
+ // response will not include form object if everything is default
+ //
+ if (vData.metadata?.form) {
+ if (vData.metadata.form?.refreshAfterSubmit)
+ refreshMode = vData.metadata.form.refreshAfterSubmit;
+ if (vData.metadata.form?.afterSubmitMessage)
+ msg = vData.metadata.form.afterSubmitMessage;
+ if (vData.metadata.form?.description)
+ desc = vData.metadata.form.description;
+ }
+
+ const formData = {
+ title: viewName,
+ heading: viewName,
+ subheading: desc,
+ success_msg: msg,
+ submit_another_form: refreshMode.includes('REFRESH_BUTTON'),
+ show_blank_form: refreshMode.includes('AUTO_REFRESH'),
+ };
+
+ logDetailed(`NC API dbView.formCreate :: ${viewName}`);
+ const _perfStart = recordPerfStart();
+ // const f = await api.dbView.formCreate(tblId, formData);
+ const f = await this.formsService.formViewCreate({
+ tableId: tblId,
+ body: formData,
+ });
+ recordPerfStats(_perfStart, 'dbView.formCreate');
+
+ logDetailed(
+ `[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${
+ formViews.length
+ }] Create ${viewName}`,
+ );
+
+ await updateNcTblSchemaById(tblId);
+
+ logDetailed(` Configure show/hide columns`);
+ await nc_configureFields(
+ f.id,
+ vData,
+ aTblSchema[idx].name,
+ viewName,
+ 'form',
+ );
+ }
+ }
+ };
+
+ const nocoConfigureGridView = async (sDB, aTblSchema) => {
+ for (let idx = 0; idx < aTblSchema.length; idx++) {
+ const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id);
+ const gridViews = aTblSchema[idx].views.filter(
+ (x) => x.type === 'grid',
+ );
+
+ let viewCnt = idx;
+ if (syncDB.options.syncViews)
+ viewCnt = rtc.view.grid + rtc.view.gallery + rtc.view.form;
+ rtc.view.grid += gridViews.length;
+
+ for (
+ let i = 0;
+ i < (sDB.options.syncViews ? gridViews.length : 1);
+ i++
+ ) {
+ logDetailed(` Axios fetch view-data`);
+ // fetch viewData JSON
+ const vData = await getViewData(gridViews[i].id);
+
+ // retrieve view name & associated NC-ID
+ const viewName = aTblSchema[idx].views.find(
+ (x) => x.id === gridViews[i].id,
+ )?.name;
+ const _perfStart = recordPerfStart();
+ // const viewList: any = await api.dbView.list(tblId);
+ const viewList = { list: [] };
+ viewList['list'] = await this.viewsService.viewList({
+ tableId: tblId,
+ user: { roles: userRole },
+ });
+ recordPerfStats(_perfStart, 'dbView.list');
+
+ let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id;
+
+ logBasic(
+ `:: [${viewCnt + i + 1}/${rtc.view.total}] Grid : ${
+ aTblSchema[idx].name
+ } / ${viewName}`,
+ );
+
+ // create view (default already created)
+ if (i > 0) {
+ logDetailed(`NC API dbView.gridCreate :: ${viewName}`);
+ const _perfStart = recordPerfStart();
+ const viewCreated = await this.gridsService.gridViewCreate({
+ tableId: tblId,
+ grid: {
+ title: viewName,
+ },
+ });
+ recordPerfStats(_perfStart, 'dbView.gridCreate');
+
+ await updateNcTblSchemaById(tblId);
+ await sMap.addToMappingTbl(
+ gridViews[i].id,
+ viewCreated.id,
+ viewName,
+ tblId,
+ );
+ ncViewId = viewCreated.id;
+ }
+
+ logDetailed(` Configure show/hide columns`);
+ await nc_configureFields(
+ ncViewId,
+ vData,
+ aTblSchema[idx].name,
+ viewName,
+ 'grid',
+ );
+
+ // configure filters
+ if (vData?.filters) {
+ logDetailed(` Configure filter set`);
+
+ // skip filters if nested
+ if (!vData.filters.filterSet.find((x) => x?.type === 'nested')) {
+ await nc_configureFilters(ncViewId, vData.filters);
+ }
+ }
+
+ // configure sort
+ if (vData?.lastSortsApplied?.sortSet.length) {
+ logDetailed(` Configure sort set`);
+ await nc_configureSort(ncViewId, vData.lastSortsApplied);
+ }
+ }
+ }
+ };
+
+ const nocoAddUsers = async (aTblSchema) => {
+ const userRoles = {
+ owner: 'owner',
+ create: 'creator',
+ edit: 'editor',
+ comment: 'commenter',
+ read: 'viewer',
+ none: 'viewer',
+ };
+ const userList = aTblSchema.appBlanket.userInfoById;
+ const totalUsers = Object.keys(userList).length;
+ let cnt = 0;
+ const insertJobs: Promise[] = [];
+
+ for (const [, value] of Object.entries(
+ userList as { [key: string]: any },
+ )) {
+ logDetailed(
+ `[${++cnt}/${totalUsers}] NC API auth.projectUserAdd :: ${
+ value.email
+ }`,
+ );
+ const _perfStart = recordPerfStart();
+ insertJobs.push(
+ this.projectUsersService
+ .userInvite({
+ projectId: ncCreatedProjectSchema.id,
+ projectUser: {
+ email: value.email,
+ roles: userRoles[value.permissionLevel],
+ },
+ req: { user: syncDB.user, clientIp: '' },
+ })
+ .catch((e) =>
+ e.response?.data?.msg
+ ? logBasic(`NOTICE: ${e.response.data.msg}`)
+ : console.log(e),
+ ),
+ );
+ recordPerfStats(_perfStart, 'auth.projectUserAdd');
+ }
+ await Promise.all(insertJobs);
+ };
+
+ const updateNcTblSchema = (tblSchema) => {
+ const tblId = tblSchema.id;
+
+ // replace entry from array if already exists
+ const idx = ncSchema.tables.findIndex((x) => x.id === tblId);
+ if (idx !== -1) ncSchema.tables.splice(idx, 1);
+ ncSchema.tables.push(tblSchema);
+
+ // overwrite object if it exists
+ ncSchema.tablesById[tblId] = tblSchema;
+ };
+
+ const updateNcTblSchemaById = async (tblId) => {
+ const _perfStart = recordPerfStart();
+ const ncTbl: any = await this.tablesService.getTableWithAccessibleViews({
+ tableId: tblId,
+ user: syncDB.user,
+ });
+ recordPerfStats(_perfStart, 'dbTable.read');
+
+ updateNcTblSchema(ncTbl);
+ };
+
+ ///////////////////////
+
+ // statistics
+ //
+ const migrationStats = [];
+
+ const generateMigrationStats = async (aTblSchema) => {
+ const migrationStatsObj = {
+ table_name: '',
+ aTbl: {
+ columns: 0,
+ links: 0,
+ lookup: 0,
+ rollup: 0,
+ },
+ nc: {
+ columns: 0,
+ links: 0,
+ lookup: 0,
+ rollup: 0,
+ invalidColumn: 0,
+ },
+ };
+ for (let idx = 0; idx < aTblSchema.length; idx++) {
+ migrationStatsObj.table_name = aTblSchema[idx].name;
+
+ const aTblLinkColumns = aTblSchema[idx].columns.filter(
+ (x) => x.type === 'foreignKey',
+ );
+ const aTblLookup = aTblSchema[idx].columns.filter(
+ (x) => x.type === 'lookup',
+ );
+ const aTblRollup = aTblSchema[idx].columns.filter(
+ (x) => x.type === 'rollup',
+ );
+
+ let invalidColumnId = 0;
+ for (let i = 0; i < aTblLookup.length; i++) {
+ if (
+ aTblLookup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length
+ ) {
+ invalidColumnId++;
+ }
+ }
+ for (let i = 0; i < aTblRollup.length; i++) {
+ if (
+ aTblRollup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length
+ ) {
+ invalidColumnId++;
+ }
+ }
+
+ migrationStatsObj.aTbl.columns = aTblSchema[idx].columns.length;
+ migrationStatsObj.aTbl.links = aTblLinkColumns.length;
+ migrationStatsObj.aTbl.lookup = aTblLookup.length;
+ migrationStatsObj.aTbl.rollup = aTblRollup.length;
+
+ const ncTbl = await nc_getTableSchema(aTblSchema[idx].name);
+ const linkColumn = ncTbl.columns.filter(
+ (x) => x.uidt === UITypes.LinkToAnotherRecord,
+ );
+ const lookup = ncTbl.columns.filter((x) => x.uidt === UITypes.Lookup);
+ const rollup = ncTbl.columns.filter((x) => x.uidt === UITypes.Rollup);
+
+ // all links hardwired as m2m. m2m generates additional tables per link
+ // hence link/2
+ migrationStatsObj.nc.columns =
+ ncTbl.columns.length - linkColumn.length / 2;
+ migrationStatsObj.nc.links = linkColumn.length / 2;
+ migrationStatsObj.nc.lookup = lookup.length;
+ migrationStatsObj.nc.rollup = rollup.length;
+ migrationStatsObj.nc.invalidColumn = invalidColumnId;
+
+ const temp = JSON.parse(JSON.stringify(migrationStatsObj));
+ migrationStats.push(temp);
+ }
+
+ const columnSum = migrationStats.reduce((accumulator, object) => {
+ return accumulator + object.nc.columns;
+ }, 0);
+ const linkSum = migrationStats.reduce((accumulator, object) => {
+ return accumulator + object.nc.links;
+ }, 0);
+ const lookupSum = migrationStats.reduce((accumulator, object) => {
+ return accumulator + object.nc.lookup;
+ }, 0);
+ const rollupSum = migrationStats.reduce((accumulator, object) => {
+ return accumulator + object.nc.rollup;
+ }, 0);
+
+ logBasic(`Quick Summary:`);
+ logBasic(`:: Total Tables: ${aTblSchema.length}`);
+ logBasic(`:: Total Columns: ${columnSum}`);
+ logBasic(`:: Links: ${linkSum}`);
+ logBasic(`:: Lookup: ${lookupSum}`);
+ logBasic(`:: Rollup: ${rollupSum}`);
+ logBasic(`:: Total Filters: ${rtc.filter}`);
+ logBasic(`:: Total Sort: ${rtc.sort}`);
+ logBasic(`:: Total Views: ${rtc.view.total}`);
+ logBasic(`:: Grid: ${rtc.view.grid}`);
+ logBasic(`:: Gallery: ${rtc.view.gallery}`);
+ logBasic(`:: Form: ${rtc.view.form}`);
+ logBasic(`:: Total Records: ${rtc.data.records}`);
+ logBasic(`:: Total Nested Links: ${rtc.data.nestedLinks}`);
+
+ const duration = Date.now() - start;
+ logBasic(`:: Migration time: ${duration}`);
+ logBasic(`:: Axios fetch count: ${rtc.fetchAt.count}`);
+ logBasic(`:: Axios fetch time: ${rtc.fetchAt.time}`);
+
+ if (debugMode) {
+ await writeJsonFileAsync('stats.json', perfStats, { spaces: 2 });
+ const perflog = [];
+ for (let i = 0; i < perfStats.length; i++) {
+ perflog.push(`${perfStats[i].e}, ${perfStats[i].d}`);
+ }
+ await writeJsonFileAsync('stats.csv', perflog, { spaces: 2 });
+ await writeJsonFileAsync('skip.txt', rtc.migrationSkipLog.log, {
+ spaces: 2,
+ });
+ }
+
+ T.event({
+ event: 'a:airtable-import:success',
+ data: {
+ stats: {
+ migrationTime: duration,
+ totalTables: aTblSchema.length,
+ totalColumns: columnSum,
+ links: linkSum,
+ lookup: lookupSum,
+ rollup: rollupSum,
+ totalFilters: rtc.filter,
+ totalSort: rtc.sort,
+ view: {
+ total: rtc.view.total,
+ grid: rtc.view.grid,
+ gallery: rtc.view.gallery,
+ form: rtc.view.form,
+ },
+ axios: {
+ count: rtc.fetchAt.count,
+ time: rtc.fetchAt.time,
+ },
+ totalRecords: rtc.data.records,
+ nestedLinks: rtc.data.nestedLinks,
+ },
+ },
+ });
+ };
+
+ //////////////////////////////
+ // filters
+
+ const filterMap = {
+ '=': 'eq',
+ '!=': 'neq',
+ '<': 'lt',
+ '<=': 'lte',
+ '>': 'gt',
+ '>=': 'gte',
+ isEmpty: 'empty',
+ isNotEmpty: 'notempty',
+ contains: 'like',
+ doesNotContain: 'nlike',
+ isAnyOf: 'anyof',
+ isNoneOf: 'nanyof',
+ '|': 'anyof',
+ '&': 'allof',
+ };
+
+ const nc_configureFilters = async (viewId, f) => {
+ for (let i = 0; i < f.filterSet.length; i++) {
+ const filter = f.filterSet[i];
+ const colSchema = await nc_getColumnSchema(filter.columnId);
+
+ // column not available;
+ // one of not migrated column;
+ if (!colSchema) {
+ updateMigrationSkipLog(
+ await sMap.getNcNameFromAtId(viewId),
+ colSchema.title,
+ colSchema.uidt,
+ `filter config skipped; column not migrated`,
+ );
+ continue;
+ }
+ const columnId = colSchema.id;
+ const datatype = colSchema.uidt;
+ const ncFilters = [];
+
+ // console.log(filter)
+ if (datatype === UITypes.Date || datatype === UITypes.DateTime) {
+ // skip filters over data datatype
+ updateMigrationSkipLog(
+ await sMap.getNcNameFromAtId(viewId),
+ colSchema.title,
+ colSchema.uidt,
+ `filter config skipped; filter over date datatype not supported`,
+ );
+ continue;
+ }
+
+ // single-select & multi-select
+ else if (
+ datatype === UITypes.SingleSelect ||
+ datatype === UITypes.MultiSelect
+ ) {
+ if (filter.operator === 'doesNotContain') {
+ filter.operator = 'isNoneOf';
+ }
+ // if array, break it down to multiple filters
+ if (Array.isArray(filter.value)) {
+ const fx = {
+ fk_column_id: columnId,
+ logical_op: f.conjunction,
+ comparison_op: filterMap[filter.operator],
+ value: (
+ await Promise.all(
+ filter.value.map(
+ async (f) => await sMap.getNcNameFromAtId(f),
+ ),
+ )
+ ).join(','),
+ };
+ ncFilters.push(fx);
+ }
+ // not array - add as is
+ else if (filter.value) {
+ const fx = {
+ fk_column_id: columnId,
+ logical_op: f.conjunction,
+ comparison_op: filterMap[filter.operator],
+ value: await sMap.getNcNameFromAtId(filter.value),
+ };
+ ncFilters.push(fx);
+ }
+ }
+
+ // other data types (number/ text/ long text/ ..)
+ else if (filter.value) {
+ const fx = {
+ fk_column_id: columnId,
+ logical_op: f.conjunction,
+ comparison_op: filterMap[filter.operator],
+ value: filter.value,
+ };
+ ncFilters.push(fx);
+ }
+
+ // insert filters
+ for (let i = 0; i < ncFilters.length; i++) {
+ const _perfStart = recordPerfStart();
+ await this.filtersService.filterCreate({
+ viewId: viewId,
+ filter: ncFilters[i],
+ });
+ recordPerfStats(_perfStart, 'dbTableFilter.create');
+
+ rtc.filter++;
+ }
+ }
+ };
+
+ const nc_configureSort = async (viewId, s) => {
+ for (let i = 0; i < s.sortSet.length; i++) {
+ const columnId = (await nc_getColumnSchema(s.sortSet[i].columnId))?.id;
+
+ if (columnId) {
+ const _perfStart = recordPerfStart();
+ await this.sortsService.sortCreate({
+ viewId: viewId,
+ sort: {
+ fk_column_id: columnId,
+ direction: s.sortSet[i].ascending ? 'asc' : 'desc',
+ },
+ });
+ recordPerfStats(_perfStart, 'dbTableSort.create');
+ }
+ rtc.sort++;
+ }
+ };
+
+ const nc_configureFields = async (
+ _viewId,
+ _c,
+ tblName,
+ viewName,
+ viewType?,
+ ) => {
+ // force hide PK column
+ const hiddenColumns = [ncSysFields.id, ncSysFields.hash];
+ const c = _c.columnOrder;
+
+ // column order corrections
+ // retrieve table schema
+ const ncTbl = await nc_getTableSchema(tblName);
+ // retrieve view ID
+ const viewId = ncTbl.views.find((x) => x.title === viewName).id;
+ let viewDetails;
+
+ const _perfStart = recordPerfStart();
+ if (viewType === 'form') {
+ viewDetails = (
+ await this.formsService.formViewGet({ formViewId: viewId })
+ ).columns;
+ recordPerfStats(_perfStart, 'dbView.formRead');
+ } else if (viewType === 'gallery') {
+ viewDetails = (
+ await this.galleriesService.galleryViewGet({
+ galleryViewId: viewId,
+ })
+ ).columns;
+ recordPerfStats(_perfStart, 'dbView.galleryRead');
+ } else {
+ viewDetails = await this.viewColumnsService.columnList({
+ viewId: viewId,
+ });
+ recordPerfStats(_perfStart, 'dbView.gridColumnsList');
+ }
+
+ // nc-specific columns; default hide.
+ for (let j = 0; j < hiddenColumns.length; j++) {
+ const ncColumnId = ncTbl.columns.find(
+ (x) => x.title === hiddenColumns[j],
+ ).id;
+ const ncViewColumnId = viewDetails.find(
+ (x) => x.fk_column_id === ncColumnId,
+ )?.id;
+ if (ncViewColumnId === undefined) continue;
+
+ // first two positions held by record id & record hash
+ const _perfStart = recordPerfStart();
+ await this.viewColumnsService.columnUpdate({
+ viewId: viewId,
+ columnId: ncViewColumnId,
+ column: {
+ show: false,
+ order: j + 1 + c.length,
+ },
+ });
+ recordPerfStats(_perfStart, 'dbViewColumn.update');
+ }
+
+ // rest of the columns from airtable- retain order & visibility property
+ for (let j = 0; j < c.length; j++) {
+ const ncColumnId = await sMap.getNcIdFromAtId(c[j].columnId);
+ const ncViewColumnId = await nc_getViewColumnId(
+ viewId,
+ viewType,
+ ncColumnId,
+ );
+ if (ncViewColumnId === undefined) continue;
+
+ // first two positions held by record id & record hash
+ const configData = { show: c[j].visibility, order: j + 1 };
+ if (viewType === 'form') {
+ if (_c?.metadata?.form?.fieldsByColumnId?.[c[j].columnId]) {
+ const x = _c.metadata.form.fieldsByColumnId[c[j].columnId];
+ const formData = { ...configData };
+ if (x?.title) formData[`label`] = x.title;
+ if (x?.required) formData[`required`] = x.required;
+ if (x?.description) formData[`description`] = x.description;
+ const _perfStart = recordPerfStart();
+ await this.formColumnsService.columnUpdate({
+ formViewColumnId: ncViewColumnId,
+ formViewColumn: formData,
+ });
+ recordPerfStats(_perfStart, 'dbView.formColumnUpdate');
+ }
+ }
+ const _perfStart = recordPerfStart();
+ await this.viewColumnsService.columnUpdate({
+ viewId: viewId,
+ columnId: ncViewColumnId,
+ column: configData,
+ });
+ recordPerfStats(_perfStart, 'dbViewColumn.update');
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////////
+ let recordCnt = 0;
+ try {
+ logBasic('SDK initialized');
+ logDetailed('Project initialization started');
+ // delete project if already exists
+ if (debugMode) await init(syncDB);
+
+ logDetailed('Project initialized');
+
+ logBasic('Retrieving Airtable schema');
+ // read schema file
+ const schema = await getAirtableSchema(syncDB);
+ const aTblSchema = schema.tableSchemas;
+ logDetailed('Project schema extraction completed');
+
+ if (!syncDB.projectId) {
+ if (!syncDB.projectName)
+ throw new Error('Project name or id not provided');
+ // create empty project
+ await nocoCreateProject(syncDB.projectName);
+ logDetailed('Project created');
+ } else {
+ await nocoGetProject(syncDB.projectId);
+ syncDB.projectName = ncCreatedProjectSchema?.title;
+ syncDB.baseId = syncDB.baseId || ncCreatedProjectSchema.bases[0].id;
+ logDetailed('Getting existing project meta');
+ }
+
+ logBasic('Importing Tables...');
+ // prepare table schema (base)
+ await nocoCreateBaseSchema(aTblSchema);
+ logDetailed('Table creation completed');
+
+ logDetailed('Configuring Links');
+ // add LTAR
+ await nocoCreateLinkToAnotherRecord(aTblSchema);
+ logDetailed('Migrating LTAR columns completed');
+
+ if (syncDB.options.syncLookup) {
+ logDetailed(`Configuring Lookup`);
+ // add look-ups
+ await nocoCreateLookups(aTblSchema);
+ logDetailed('Migrating Lookup columns completed');
+ }
+
+ if (syncDB.options.syncRollup) {
+ logDetailed('Configuring Rollup');
+ // add roll-ups
+ await nocoCreateRollup(aTblSchema);
+ logDetailed('Migrating Rollup columns completed');
+
+ if (syncDB.options.syncLookup) {
+ logDetailed('Migrating Lookup form Rollup columns');
+ // lookups for rollup
+ await nocoLookupForRollup();
+ logDetailed('Migrating Lookup form Rollup columns completed');
+ }
+ }
+ logDetailed('Configuring Display Value column');
+ // configure Display Value
+ await nocoSetPrimary(aTblSchema);
+ logDetailed('Configuring Display Value column completed');
+
+ logBasic('Configuring User(s)');
+ // add users
+ await nocoAddUsers(schema);
+ logDetailed('Adding users completed');
+
+ // hide-fields
+ // await nocoReconfigureFields(aTblSchema);
+
+ logBasic('Syncing views');
+ // configure views
+ await nocoConfigureGridView(syncDB, aTblSchema);
+ await nocoConfigureFormView(syncDB, aTblSchema);
+ await nocoConfigureGalleryView(syncDB, aTblSchema);
+ logDetailed('Syncing views completed');
+
+ if (syncDB.options.syncData) {
+ try {
+ // await nc_DumpTableSchema();
+ const _perfStart = recordPerfStart();
+ const ncTblList = { list: [] };
+ ncTblList['list'] = await this.tablesService.getAccessibleTables({
+ projectId: ncCreatedProjectSchema.id,
+ baseId: syncDB.baseId,
+ roles: userRole,
+ });
+ recordPerfStats(_perfStart, 'base.tableList');
+
+ logBasic('Reading Records...');
+
+ const recordsMap = {};
+
+ for (let i = 0; i < ncTblList.list.length; i++) {
+ // not a migrated table, skip
+ if (
+ undefined ===
+ aTblSchema.find((x) => x.name === ncTblList.list[i].title)
+ )
+ continue;
+
+ const _perfStart = recordPerfStart();
+ const ncTbl: any =
+ await this.tablesService.getTableWithAccessibleViews({
+ tableId: ncTblList.list[i].id,
+ user: syncDB.user,
+ });
+ recordPerfStats(_perfStart, 'dbTable.read');
+
+ recordCnt = 0;
+
+ recordsMap[ncTbl.id] = await importData({
+ projectName: syncDB.projectName,
+ table: ncTbl,
+ base,
+ logBasic,
+ nocoBaseDataProcessing_v2,
+ sDB: syncDB,
+ logDetailed,
+ services: {
+ tableService: this.tablesService,
+ bulkDataService: this.bulkDataAliasService,
+ },
+ });
+ rtc.data.records += await recordsMap[ncTbl.id].getCount();
+
+ logDetailed(`Data inserted from ${ncTbl.title}`);
+ }
+
+ logBasic('Configuring Record Links...');
+ for (let i = 0; i < ncTblList.list.length; i++) {
+ // not a migrated table, skip
+ if (
+ undefined ===
+ aTblSchema.find((x) => x.name === ncTblList.list[i].title)
+ )
+ continue;
+
+ // const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
+ const ncTbl: any =
+ await this.tablesService.getTableWithAccessibleViews({
+ tableId: ncTblList.list[i].id,
+ user: syncDB.user,
+ });
+
+ rtc.data.nestedLinks += await importLTARData({
+ table: ncTbl,
+ projectName: syncDB.projectName,
+ base,
+ fields: null, //Object.values(tblLinkGroup).flat(),
+ logBasic,
+ insertedAssocRef,
+ logDetailed,
+ records: recordsMap[ncTbl.id],
+ atNcAliasRef,
+ ncLinkMappingTable,
+ syncDB,
+ services: {
+ tableService: this.tablesService,
+ bulkDataService: this.bulkDataAliasService,
+ },
+ });
+ }
+ } catch (error) {
+ logDetailed(
+ `There was an error while migrating data! Please make sure your API key (${syncDB.apiKey}) is correct.`,
+ );
+ logDetailed(`Error: ${error}`);
+ }
+ }
+ if (generate_migrationStats) {
+ await generateMigrationStats(aTblSchema);
+ }
+ } catch (e) {
+ if (e.response?.data?.msg) {
+ T.event({
+ event: 'a:airtable-import:error',
+ data: { error: e.response.data.msg },
+ });
+ throw new Error(e.response.data.msg);
+ }
+ throw e;
+ }
+ }
+}
+
+export const getUniqueNameGenerator = (defaultName = 'name') => {
+ const namesRef = {};
+
+ return (initName: string = defaultName): string => {
+ let name = initName === '_' ? defaultName : initName;
+ let c = 0;
+ while (name in namesRef) {
+ name = `${initName}_${++c}`;
+ }
+ namesRef[name] = true;
+ return name;
+ };
+};
+
+export interface AirtableSyncConfig {
+ syncId: string;
+ baseURL: string;
+ authToken: string;
+ projectName?: string;
+ projectId?: string;
+ baseId?: string;
+ apiKey: string;
+ shareId: string;
+ user: UserType;
+ options: {
+ syncViews: boolean;
+ syncData: boolean;
+ syncRollup: boolean;
+ syncLookup: boolean;
+ syncFormula: boolean;
+ syncAttachment: boolean;
+ };
+}
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/helpers/EntityMap.ts b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/EntityMap.ts
new file mode 100644
index 0000000000..afbd1f0f53
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/EntityMap.ts
@@ -0,0 +1,222 @@
+import { Readable } from 'stream';
+import sqlite3 from 'sqlite3';
+
+class EntityMap {
+ initialized: boolean;
+ cols: string[];
+ db: any;
+
+ constructor(...args) {
+ this.initialized = false;
+ this.cols = args.map((arg) => processKey(arg));
+ this.db = new Promise((resolve, reject) => {
+ const db = new sqlite3.Database(':memory:');
+
+ const colStatement =
+ this.cols.length > 0
+ ? this.cols.join(' TEXT, ') + ' TEXT'
+ : 'mappingPlaceholder TEXT';
+ db.run(`CREATE TABLE mapping (${colStatement})`, (err) => {
+ if (err) {
+ console.log(err);
+ reject(err);
+ }
+ resolve(db);
+ });
+ });
+ }
+
+ async init() {
+ if (!this.initialized) {
+ this.db = await this.db;
+ this.initialized = true;
+ }
+ }
+
+ destroy() {
+ if (this.initialized && this.db) {
+ this.db.close();
+ }
+ }
+
+ async addRow(row) {
+ if (!this.initialized) {
+ throw 'Please initialize first!';
+ }
+
+ const cols = Object.keys(row).map((key) => processKey(key));
+ const colStatement = cols.map((key) => `'${key}'`).join(', ');
+ const questionMarks = cols.map(() => '?').join(', ');
+
+ const promises = [];
+
+ for (const col of cols.filter((col) => !this.cols.includes(col))) {
+ promises.push(
+ new Promise((resolve, reject) => {
+ this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => {
+ if (err) {
+ console.log(err);
+ reject(err);
+ }
+ this.cols.push(col);
+ resolve(true);
+ });
+ }),
+ );
+ }
+
+ await Promise.all(promises);
+
+ const values = Object.values(row).map((val) => {
+ if (typeof val === 'object') {
+ return `JSON::${JSON.stringify(val)}`;
+ }
+ return val;
+ });
+
+ return new Promise((resolve, reject) => {
+ this.db.run(
+ `INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`,
+ values,
+ (err) => {
+ if (err) {
+ console.log(err);
+ reject(err);
+ }
+ resolve(true);
+ },
+ );
+ });
+ }
+
+ getRow(col, val, res = []): Promise> {
+ if (!this.initialized) {
+ throw 'Please initialize first!';
+ }
+ return new Promise((resolve, reject) => {
+ col = processKey(col);
+ res = res.map((r) => processKey(r));
+ this.db.get(
+ `SELECT ${
+ res.length ? res.join(', ') : '*'
+ } FROM mapping WHERE ${col} = ?`,
+ [val],
+ (err, rs) => {
+ if (err) {
+ console.log(err);
+ reject(err);
+ }
+ if (rs) {
+ rs = processResponseRow(rs);
+ }
+ resolve(rs);
+ },
+ );
+ });
+ }
+
+ getCount(): Promise {
+ if (!this.initialized) {
+ throw 'Please initialize first!';
+ }
+ return new Promise((resolve, reject) => {
+ this.db.get(`SELECT COUNT(*) as count FROM mapping`, (err, rs) => {
+ if (err) {
+ console.log(err);
+ reject(err);
+ }
+ resolve(rs.count);
+ });
+ });
+ }
+
+ getStream(res = []): DBStream {
+ if (!this.initialized) {
+ throw 'Please initialize first!';
+ }
+ res = res.map((r) => processKey(r));
+ return new DBStream(
+ this.db,
+ `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping`,
+ );
+ }
+
+ getLimit(limit, offset, res = []): Promise[]> {
+ if (!this.initialized) {
+ throw 'Please initialize first!';
+ }
+ return new Promise((resolve, reject) => {
+ res = res.map((r) => processKey(r));
+ this.db.all(
+ `SELECT ${
+ res.length ? res.join(', ') : '*'
+ } FROM mapping LIMIT ${limit} OFFSET ${offset}`,
+ (err, rs) => {
+ if (err) {
+ console.log(err);
+ reject(err);
+ }
+ for (let row of rs) {
+ row = processResponseRow(row);
+ }
+ resolve(rs);
+ },
+ );
+ });
+ }
+}
+
+class DBStream extends Readable {
+ db: any;
+ stmt: any;
+ sql: any;
+
+ constructor(db, sql) {
+ super({ objectMode: true });
+ this.db = db;
+ this.sql = sql;
+ this.stmt = this.db.prepare(this.sql);
+ this.on('end', () => this.stmt.finalize());
+ }
+
+ _read() {
+ const stream = this;
+ this.stmt.get(function (err, result) {
+ if (err) {
+ stream.emit('error', err);
+ } else {
+ if (result) {
+ result = processResponseRow(result);
+ }
+ stream.push(result || null);
+ }
+ });
+ }
+}
+
+function processResponseRow(res: any) {
+ for (const key of Object.keys(res)) {
+ if (res[key] && res[key].startsWith('JSON::')) {
+ try {
+ res[key] = JSON.parse(res[key].replace('JSON::', ''));
+ } catch (e) {
+ console.log(e);
+ }
+ }
+ if (revertKey(key) !== key) {
+ res[revertKey(key)] = res[key];
+ delete res[key];
+ }
+ }
+ return res;
+}
+
+function processKey(key) {
+ return key.replace(/'/g, "''").replace(/[A-Z]/g, (match) => `_${match}`);
+}
+
+function revertKey(key) {
+ return key.replace(/''/g, "'").replace(/_[A-Z]/g, (match) => match[1]);
+}
+
+export default EntityMap;
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/helpers/fetchAT.ts b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/fetchAT.ts
new file mode 100644
index 0000000000..1a9a8151bb
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/fetchAT.ts
@@ -0,0 +1,242 @@
+import axios from 'axios';
+
+const info: any = {
+ initialized: false,
+};
+
+async function initialize(shareId) {
+ info.cookie = '';
+ const url = `https://airtable.com/${shareId}`;
+
+ try {
+ const hreq = await axios
+ .get(url, {
+ headers: {
+ accept:
+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'sec-ch-ua':
+ '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-ch-ua-platform': '"Linux"',
+ 'sec-fetch-dest': 'document',
+ 'sec-fetch-mode': 'navigate',
+ 'sec-fetch-site': 'none',
+ 'sec-fetch-user': '?1',
+ 'upgrade-insecure-requests': '1',
+ 'User-Agent':
+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+ },
+ // @ts-ignore
+ referrerPolicy: 'strict-origin-when-cross-origin',
+ body: null,
+ method: 'GET',
+ })
+ .then((response) => {
+ for (const ck of response.headers['set-cookie']) {
+ info.cookie += ck.split(';')[0] + '; ';
+ }
+ return response.data;
+ })
+ .catch(() => {
+ throw {
+ message:
+ 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details',
+ };
+ });
+
+ info.headers = JSON.parse(
+ hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
+ );
+ info.link = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim());
+ info.baseInfo = decodeURIComponent(info.link)
+ .match(/{(.*)}/g)[0]
+ .split('&')
+ .reduce((result, el) => {
+ try {
+ return Object.assign(
+ result,
+ JSON.parse(el.includes('=') ? el.split('=')[1] : el),
+ );
+ } catch (e) {
+ if (el.includes('=')) {
+ return Object.assign(result, {
+ [el.split('=')[0]]: el.split('=')[1],
+ });
+ }
+ }
+ }, {});
+ info.baseId = info.baseInfo.applicationId;
+ info.initialized = true;
+ } catch (e) {
+ console.log(e);
+ info.initialized = false;
+ if (e.message) {
+ throw e;
+ } else {
+ throw {
+ message:
+ 'Error processing Shared Base :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details',
+ };
+ }
+ }
+}
+
+async function read() {
+ if (info.initialized) {
+ const resreq = await axios('https://airtable.com' + info.link, {
+ headers: {
+ accept: '*/*',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'sec-ch-ua':
+ '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-ch-ua-platform': '"Linux"',
+ 'sec-fetch-dest': 'empty',
+ 'sec-fetch-mode': 'cors',
+ 'sec-fetch-site': 'same-origin',
+ 'User-Agent':
+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+ 'x-time-zone': 'Europe/Berlin',
+ cookie: info.cookie,
+ ...info.headers,
+ },
+ // @ts-ignore
+ referrerPolicy: 'no-referrer',
+ body: null,
+ method: 'GET',
+ })
+ .then((response) => {
+ return response.data;
+ })
+ .catch(() => {
+ throw {
+ message:
+ 'Error Reading :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details',
+ };
+ });
+
+ return {
+ schema: resreq.data,
+ baseId: info.baseId,
+ baseInfo: info.baseInfo,
+ };
+ } else {
+ throw {
+ message: 'Error Initializing :: please try again !!',
+ };
+ }
+}
+
+async function readView(viewId) {
+ if (info.initialized) {
+ const resreq = await axios(
+ `https://airtable.com/v0.3/view/${viewId}/readData?` +
+ `stringifiedObjectParams=${encodeURIComponent('{}')}&requestId=${
+ info.baseInfo.requestId
+ }&accessPolicy=${encodeURIComponent(
+ JSON.stringify({
+ allowedActions: info.baseInfo.allowedActions,
+ shareId: info.baseInfo.shareId,
+ applicationId: info.baseInfo.applicationId,
+ generationNumber: info.baseInfo.generationNumber,
+ expires: info.baseInfo.expires,
+ signature: info.baseInfo.signature,
+ }),
+ )}`,
+ {
+ headers: {
+ accept: '*/*',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'sec-ch-ua':
+ '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-ch-ua-platform': '"Linux"',
+ 'sec-fetch-dest': 'empty',
+ 'sec-fetch-mode': 'cors',
+ 'sec-fetch-site': 'same-origin',
+ 'User-Agent':
+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+ 'x-time-zone': 'Europe/Berlin',
+ cookie: info.cookie,
+ ...info.headers,
+ },
+ // @ts-ignore
+ referrerPolicy: 'no-referrer',
+ body: null,
+ method: 'GET',
+ },
+ )
+ .then((response) => {
+ return response.data;
+ })
+ .catch(() => {
+ throw {
+ message:
+ 'Error Reading View :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details',
+ };
+ });
+ return { view: resreq.data };
+ } else {
+ throw {
+ message: 'Error Initializing :: please try again !!',
+ };
+ }
+}
+
+async function readTemplate(templateId) {
+ if (!info.initialized) {
+ await initialize('shrO8aYf3ybwSdDKn');
+ }
+ const resreq = await axios(
+ `https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
+ {
+ headers: {
+ accept: '*/*',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'sec-ch-ua':
+ '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-ch-ua-platform': '"Linux"',
+ 'sec-fetch-dest': 'empty',
+ 'sec-fetch-mode': 'cors',
+ 'sec-fetch-site': 'same-origin',
+ 'User-Agent':
+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+ 'x-time-zone': 'Europe/Berlin',
+ cookie: info.cookie,
+ ...info.headers,
+ },
+ // @ts-ignore
+ referrer: 'https://www.airtable.com/',
+ referrerPolicy: 'same-origin',
+ body: null,
+ method: 'GET',
+ mode: 'cors',
+ credentials: 'include',
+ },
+ )
+ .then((response) => {
+ return response.data;
+ })
+ .catch(() => {
+ throw {
+ message:
+ 'Error Fetching :: Ensure www.airtable.com/templates/featured/ is accessible.',
+ };
+ });
+ return { template: resreq };
+}
+
+function unicodeToChar(text) {
+ return text.replace(/\\u[\dA-F]{4}/gi, function (match) {
+ return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16));
+ });
+}
+
+export default {
+ initialize,
+ read,
+ readView,
+ readTemplate,
+};
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/helpers/readAndProcessData.ts b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/readAndProcessData.ts
new file mode 100644
index 0000000000..7de16fad7e
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/readAndProcessData.ts
@@ -0,0 +1,362 @@
+/* eslint-disable no-async-promise-executor */
+import { RelationTypes, UITypes } from 'nocodb-sdk';
+import EntityMap from './EntityMap';
+import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
+import type { TablesService } from '../../../../services/tables.service';
+// @ts-ignore
+import type { AirtableBase } from 'airtable/lib/airtable_base';
+import type { TableType } from 'nocodb-sdk';
+
+const BULK_DATA_BATCH_SIZE = 500;
+const ASSOC_BULK_DATA_BATCH_SIZE = 1000;
+const BULK_PARALLEL_PROCESS = 5;
+
+interface AirtableImportContext {
+ bulkDataService: BulkDataAliasService;
+ tableService: TablesService;
+}
+
+async function readAllData({
+ table,
+ fields,
+ base,
+ logBasic = (_str) => {},
+ services,
+}: {
+ table: { title?: string };
+ fields?;
+ base: AirtableBase;
+ logBasic?: (string) => void;
+ logDetailed?: (string) => void;
+ services: AirtableImportContext;
+}): Promise {
+ return new Promise((resolve, reject) => {
+ let data = null;
+
+ const selectParams: any = {
+ pageSize: 100,
+ };
+
+ if (fields) selectParams.fields = fields;
+
+ base(table.title)
+ .select(selectParams)
+ .eachPage(
+ async function page(records, fetchNextPage) {
+ if (!data) {
+ data = new EntityMap();
+ await data.init();
+ }
+
+ for await (const record of records) {
+ await data.addRow({ id: record.id, ...record.fields });
+ }
+
+ const tmpLength = await data.getCount();
+
+ logBasic(
+ `:: Reading '${table.title}' data :: ${Math.max(
+ 1,
+ tmpLength - records.length,
+ )} - ${tmpLength}`,
+ );
+
+ // To fetch the next page of records, call `fetchNextPage`.
+ // If there are more records, `page` will get called again.
+ // If there are no more records, `done` will get called.
+ fetchNextPage();
+ },
+ async function done(err) {
+ if (err) {
+ console.error(err);
+ return reject(err);
+ }
+ resolve(data);
+ },
+ );
+ });
+}
+
+export async function importData({
+ projectName,
+ table,
+ base,
+ nocoBaseDataProcessing_v2,
+ sDB,
+ logDetailed = (_str) => {},
+ logBasic = (_str) => {},
+ services,
+}: {
+ projectName: string;
+ table: { title?: string; id?: string };
+ fields?;
+ base: AirtableBase;
+ logBasic: (string) => void;
+ logDetailed: (string) => void;
+ nocoBaseDataProcessing_v2;
+ sDB;
+ services: AirtableImportContext;
+}): Promise {
+ try {
+ // @ts-ignore
+ const records = await readAllData({
+ table,
+ base,
+ logDetailed,
+ logBasic,
+ });
+
+ await new Promise(async (resolve) => {
+ const readable = records.getStream();
+ const allRecordsCount = await records.getCount();
+ const promises = [];
+ let tempData = [];
+ let importedCount = 0;
+ let activeProcess = 0;
+ readable.on('data', async (record) => {
+ promises.push(
+ new Promise(async (resolve) => {
+ activeProcess++;
+ if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause();
+ const { id: rid, ...fields } = record;
+ const r = await nocoBaseDataProcessing_v2(sDB, table, {
+ id: rid,
+ fields,
+ });
+ tempData.push(r);
+
+ if (tempData.length >= BULK_DATA_BATCH_SIZE) {
+ let insertArray = tempData.splice(0, tempData.length);
+
+ await services.bulkDataService.bulkDataInsert({
+ projectName,
+ tableName: table.title,
+ body: insertArray,
+ cookie: {},
+ });
+
+ logBasic(
+ `:: Importing '${
+ table.title
+ }' data :: ${importedCount} - ${Math.min(
+ importedCount + BULK_DATA_BATCH_SIZE,
+ allRecordsCount,
+ )}`,
+ );
+ importedCount += insertArray.length;
+ insertArray = [];
+ }
+ activeProcess--;
+ if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume();
+ resolve(true);
+ }),
+ );
+ });
+ readable.on('end', async () => {
+ await Promise.all(promises);
+ if (tempData.length > 0) {
+ await services.bulkDataService.bulkDataInsert({
+ projectName,
+ tableName: table.title,
+ body: tempData,
+ cookie: {},
+ });
+
+ logBasic(
+ `:: Importing '${
+ table.title
+ }' data :: ${importedCount} - ${Math.min(
+ importedCount + BULK_DATA_BATCH_SIZE,
+ allRecordsCount,
+ )}`,
+ );
+ importedCount += tempData.length;
+ tempData = [];
+ }
+ resolve(true);
+ });
+ });
+
+ return records;
+ } catch (e) {
+ console.log(e);
+ return null;
+ }
+}
+
+export async function importLTARData({
+ table,
+ fields,
+ base,
+ projectName,
+ insertedAssocRef = {},
+ logDetailed = (_str) => {},
+ logBasic = (_str) => {},
+ records,
+ atNcAliasRef,
+ ncLinkMappingTable,
+ syncDB,
+ services,
+}: {
+ projectName: string;
+ table: { title?: string; id?: string };
+ fields;
+ base: AirtableBase;
+ logDetailed: (string) => void;
+ logBasic: (string) => void;
+ insertedAssocRef: { [assocTableId: string]: boolean };
+ records?: EntityMap;
+ atNcAliasRef: {
+ [ncTableId: string]: {
+ [ncTitle: string]: string;
+ };
+ };
+ ncLinkMappingTable: Record>[];
+ syncDB;
+ services: AirtableImportContext;
+}) {
+ const assocTableMetas: Array<{
+ modelMeta: { id?: string; title?: string };
+ colMeta: { title?: string };
+ curCol: { title?: string };
+ refCol: { title?: string };
+ }> = [];
+ const allData =
+ records ||
+ (await readAllData({
+ table,
+ fields,
+ base,
+ logDetailed,
+ logBasic,
+ services,
+ }));
+
+ const modelMeta: any =
+ await services.tableService.getTableWithAccessibleViews({
+ tableId: table.id,
+ user: syncDB.user,
+ });
+
+ for (const colMeta of modelMeta.columns) {
+ // skip columns which are not LTAR and Many to many
+ if (
+ colMeta.uidt !== UITypes.LinkToAnotherRecord ||
+ colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY
+ ) {
+ continue;
+ }
+
+ // skip if already inserted
+ if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue;
+
+ // self links: skip if the column under consideration is the add-on column NocoDB creates
+ if (ncLinkMappingTable.every((a) => a.nc.title !== colMeta.title)) continue;
+
+ // mark as inserted
+ insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true;
+
+ const assocModelMeta: TableType =
+ (await services.tableService.getTableWithAccessibleViews({
+ tableId: colMeta.colOptions.fk_mm_model_id,
+ user: syncDB.user,
+ })) as any;
+
+ // extract associative table and columns meta
+ assocTableMetas.push({
+ modelMeta: assocModelMeta,
+ colMeta,
+ curCol: assocModelMeta.columns.find(
+ (c) => c.id === colMeta.colOptions.fk_mm_child_column_id,
+ ),
+ refCol: assocModelMeta.columns.find(
+ (c) => c.id === colMeta.colOptions.fk_mm_parent_column_id,
+ ),
+ });
+ }
+
+ let nestedLinkCnt = 0;
+ // Iterate over all related M2M associative table
+ for await (const assocMeta of assocTableMetas) {
+ let assocTableData = [];
+ let importedCount = 0;
+
+ // extract insert data from records
+ await new Promise((resolve) => {
+ const promises = [];
+ const readable = allData.getStream();
+ let activeProcess = 0;
+ readable.on('data', async (record) => {
+ promises.push(
+ new Promise(async (resolve) => {
+ activeProcess++;
+ if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause();
+ const { id: _atId, ...rec } = record;
+
+ // todo: use actual alias instead of sanitized
+ assocTableData.push(
+ ...(
+ rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || []
+ ).map((id) => ({
+ [assocMeta.curCol.title]: record.id,
+ [assocMeta.refCol.title]: id,
+ })),
+ );
+
+ if (assocTableData.length >= ASSOC_BULK_DATA_BATCH_SIZE) {
+ let insertArray = assocTableData.splice(0, assocTableData.length);
+ logBasic(
+ `:: Importing '${
+ table.title
+ }' LTAR data :: ${importedCount} - ${Math.min(
+ importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
+ insertArray.length,
+ )}`,
+ );
+
+ await services.bulkDataService.bulkDataInsert({
+ projectName,
+ tableName: assocMeta.modelMeta.title,
+ body: insertArray,
+ cookie: {},
+ });
+
+ importedCount += insertArray.length;
+ insertArray = [];
+ }
+ activeProcess--;
+ if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume();
+ resolve(true);
+ }),
+ );
+ });
+ readable.on('end', async () => {
+ await Promise.all(promises);
+ if (assocTableData.length >= 0) {
+ logBasic(
+ `:: Importing '${
+ table.title
+ }' LTAR data :: ${importedCount} - ${Math.min(
+ importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
+ assocTableData.length,
+ )}`,
+ );
+
+ await services.bulkDataService.bulkDataInsert({
+ projectName,
+ tableName: assocMeta.modelMeta.title,
+ body: assocTableData,
+ cookie: {},
+ });
+
+ importedCount += assocTableData.length;
+ assocTableData = [];
+ }
+ resolve(true);
+ });
+ });
+
+ nestedLinkCnt += importedCount;
+ }
+ return nestedLinkCnt;
+}
diff --git a/packages/nocodb-nest/src/modules/jobs/at-import/helpers/syncMap.ts b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/syncMap.ts
new file mode 100644
index 0000000000..79ea61511c
--- /dev/null
+++ b/packages/nocodb-nest/src/modules/jobs/at-import/helpers/syncMap.ts
@@ -0,0 +1,31 @@
+export const mapTbl = {};
+
+// static mapping records between aTblId && ncId
+export const addToMappingTbl = function addToMappingTbl(
+ aTblId,
+ ncId,
+ ncName,
+ parent?,
+) {
+ mapTbl[aTblId] = {
+ ncId: ncId,
+ ncParent: parent,
+ // name added to assist in quick debug
+ ncName: ncName,
+ };
+};
+
+// get NcID from airtable ID
+export const getNcIdFromAtId = function getNcIdFromAtId(aId) {
+ return mapTbl[aId]?.ncId;
+};
+
+// get nc Parent from airtable ID
+export const getNcParentFromAtId = function getNcParentFromAtId(aId) {
+ return mapTbl[aId]?.ncParent;
+};
+
+// get nc-title from airtable ID
+export const getNcNameFromAtId = function getNcNameFromAtId(aId) {
+ return mapTbl[aId]?.ncName;
+};
diff --git a/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts b/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts
index 3ac9ced9ff..bd3c9ee4c5 100644
--- a/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts
+++ b/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts
@@ -3,6 +3,7 @@ import PQueue from 'p-queue';
import Emittery from 'emittery';
import { DuplicateProcessor } from './export-import/duplicate.processor';
import { JobsEventService } from './jobs-event.service';
+import { AtImportProcessor } from './at-import/at-import.processor';
interface Job {
id: string;
@@ -22,6 +23,7 @@ export class QueueService {
constructor(
private readonly jobsEventService: JobsEventService,
private readonly duplicateProcessor: DuplicateProcessor,
+ private readonly atImportProcessor: AtImportProcessor,
) {
this.emitter.on('active', (data: any) => {
const job = this.queueMemory.find(
@@ -56,6 +58,10 @@ export class QueueService {
this: this.duplicateProcessor,
fn: this.duplicateProcessor.duplicateBase,
},
+ 'at-import': {
+ this: this.atImportProcessor,
+ fn: this.atImportProcessor.job,
+ },
};
async jobWrapper(job: Job) {
diff --git a/packages/nocodb-nest/src/modules/jobs/jobs-event.service.ts b/packages/nocodb-nest/src/modules/jobs/jobs-event.service.ts
index dfc400e4e7..106fc1547f 100644
--- a/packages/nocodb-nest/src/modules/jobs/jobs-event.service.ts
+++ b/packages/nocodb-nest/src/modules/jobs/jobs-event.service.ts
@@ -38,6 +38,7 @@ export class JobsEventService {
name: job.name,
id: job.id.toString(),
status: 'failed',
+ error: error?.message,
});
}
diff --git a/packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts b/packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts
index f9b521afa1..c7c3420777 100644
--- a/packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts
+++ b/packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts
@@ -40,10 +40,16 @@ export class JobsGateway implements OnModuleInit {
@SubscribeMessage('subscribe')
async subscribe(
- @MessageBody() data: { name: string; id: string } | { id: string },
+ @MessageBody()
+ body: { _id: number; data: { id: string; name: string } | any },
@ConnectedSocket() client: Socket,
): Promise {
- if ('name' in data) {
+ const { _id, data } = body;
+ if (
+ Object.keys(data).every((k) => ['name', 'id'].includes(k)) &&
+ data?.name &&
+ data?.id
+ ) {
const rooms = (await this.jobsService.jobList(data.name)).map(
(j) => `${j.name}-${j.id}`,
);
@@ -51,17 +57,17 @@ export class JobsGateway implements OnModuleInit {
if (room) {
client.join(`${data.name}-${data.id}`);
client.emit('subscribed', {
- subbed: data.id,
+ _id,
name: data.name,
id: data.id,
});
}
} else {
- const job = await this.jobsService.getJobWithData({ id: data.id });
+ const job = await this.jobsService.getJobWithData(data);
if (job) {
client.join(`${job.name}-${job.id}`);
client.emit('subscribed', {
- subbed: data.id,
+ _id,
name: job.name,
id: job.id,
});
@@ -71,10 +77,12 @@ export class JobsGateway implements OnModuleInit {
@SubscribeMessage('status')
async status(
- @MessageBody() data: { name: string; id: string },
+ @MessageBody() body: { _id: number; data: { id: string; name: string } },
@ConnectedSocket() client: Socket,
): Promise {
+ const { _id, data } = body;
client.emit('status', {
+ _id,
id: data.id,
name: data.name,
status: await this.jobsService.jobStatus(data.id),
@@ -93,11 +101,13 @@ export class JobsGateway implements OnModuleInit {
| 'failed'
| 'paused'
| 'refresh';
+ error?: any;
}): Promise {
this.server.to(`${data.name}-${data.id}`).emit('status', {
id: data.id,
name: data.name,
status: data.status,
+ error: data.error,
});
}
diff --git a/packages/nocodb-nest/src/modules/jobs/jobs.module.ts b/packages/nocodb-nest/src/modules/jobs/jobs.module.ts
index 33c8b81471..13de39a065 100644
--- a/packages/nocodb-nest/src/modules/jobs/jobs.module.ts
+++ b/packages/nocodb-nest/src/modules/jobs/jobs.module.ts
@@ -11,6 +11,8 @@ import { DuplicateProcessor } from './export-import/duplicate.processor';
import { JobsGateway } from './jobs.gateway';
import { QueueService } from './fallback-queue.service';
import { JobsEventService } from './jobs-event.service';
+import { AtImportController } from './at-import/at-import.controller';
+import { AtImportProcessor } from './at-import/at-import.processor';
@Module({
imports: [
@@ -21,7 +23,7 @@ import { JobsEventService } from './jobs-event.service';
name: 'jobs',
}),
],
- controllers: [DuplicateController],
+ controllers: [DuplicateController, AtImportController],
providers: [
QueueService,
JobsGateway,
@@ -30,6 +32,7 @@ import { JobsEventService } from './jobs-event.service';
DuplicateProcessor,
ExportService,
ImportService,
+ AtImportProcessor,
],
})
export class JobsModule {}
diff --git a/packages/nocodb-nest/src/modules/jobs/jobs.service.ts b/packages/nocodb-nest/src/modules/jobs/jobs.service.ts
index 3271f631bb..dc0abed3c2 100644
--- a/packages/nocodb-nest/src/modules/jobs/jobs.service.ts
+++ b/packages/nocodb-nest/src/modules/jobs/jobs.service.ts
@@ -21,17 +21,17 @@ export class JobsService {
async jobList(jobType: string) {
return (
- await this.activeQueue.getJobs(['active', 'waiting', 'delayed'])
+ await this.activeQueue.getJobs(['active', 'waiting', 'delayed', 'paused'])
).filter((j) => j.name === jobType);
}
async getJobWithData(data: any) {
const jobs = await this.activeQueue.getJobs([
- 'completed',
+ // 'completed',
'waiting',
'active',
'delayed',
- 'failed',
+ // 'failed',
'paused',
]);
diff --git a/packages/nocodb-nest/src/modules/metas/metas.module.ts b/packages/nocodb-nest/src/modules/metas/metas.module.ts
index 1753b278e3..75f96f75cc 100644
--- a/packages/nocodb-nest/src/modules/metas/metas.module.ts
+++ b/packages/nocodb-nest/src/modules/metas/metas.module.ts
@@ -16,7 +16,6 @@ import { GalleriesController } from '../../controllers/galleries.controller';
import { GridColumnsController } from '../../controllers/grid-columns.controller';
import { GridsController } from '../../controllers/grids.controller';
import { HooksController } from '../../controllers/hooks.controller';
-import { ImportController } from '../../controllers/imports/import.controller';
import { KanbansController } from '../../controllers/kanbans.controller';
import { MapsController } from '../../controllers/maps.controller';
import { MetaDiffsController } from '../../controllers/meta-diffs.controller';
@@ -98,7 +97,6 @@ import { DatasModule } from '../datas/datas.module';
GridColumnsController,
GridsController,
HooksController,
- ImportController,
KanbansController,
MapsController,
MetaDiffsController,
@@ -170,6 +168,8 @@ import { DatasModule } from '../datas/datas.module';
GalleriesService,
KanbansService,
ProjectsService,
+ AttachmentsService,
+ ProjectUsersService,
],
})
export class MetasModule {}
diff --git a/packages/nocodb-nest/src/services/socket.gateway.ts b/packages/nocodb-nest/src/services/socket.gateway.ts
index c621847de6..192c24caaa 100644
--- a/packages/nocodb-nest/src/services/socket.gateway.ts
+++ b/packages/nocodb-nest/src/services/socket.gateway.ts
@@ -25,7 +25,6 @@ function getHash(str) {
export class SocketGateway implements OnModuleInit {
// private server: HttpServer;
private clients: { [id: string]: Socket } = {};
- private _jobs: { [id: string]: { last_message: any } } = {};
constructor(
private jwtStrategy: JwtStrategy,
@@ -59,21 +58,10 @@ export class SocketGateway implements OnModuleInit {
socket.on('event', (args) => {
T.event({ ...args, id });
});
- socket.on('subscribe', (room) => {
- if (room in this.jobs) {
- socket.join(room);
- socket.emit('job');
- socket.emit('progress', this.jobs[room].last_message);
- }
- });
});
}
public get io() {
return this.server;
}
-
- public get jobs() {
- return this._jobs;
- }
}